web-cortex-framework 2.1.1__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/Cargo.lock +2 -2
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/Cargo.toml +1 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/PKG-INFO +48 -4
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/README.md +47 -3
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent/provider.rs +70 -14
- web_cortex_framework-2.2.0/crates/webcortex-core/src/agent/vision.rs +348 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent.rs +222 -14
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/app.rs +70 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/manifest.rs +19 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/mcp.rs +28 -2
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/server.rs +24 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-py/src/behaviour.rs +2 -4
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/pyproject.toml +1 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/__init__.py +3 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/_bridge.py +4 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/app.py +74 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/cli.py +55 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/contextpack.py +18 -3
- web_cortex_framework-2.2.0/python/webcortex/media.py +240 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/starters.py +206 -2
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/LICENSE +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/Cargo.toml +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent/registry.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent/session.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/audit.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/auth.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/bridge.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/context.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/db.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/files.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/flow.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/http.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/ledger.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/lib.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/middleware.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/openapi.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/router.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/templates.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/typegen.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-py/Cargo.toml +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-py/src/lib.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/schema.py +0 -0
|
@@ -2616,7 +2616,7 @@ dependencies = [
|
|
|
2616
2616
|
|
|
2617
2617
|
[[package]]
|
|
2618
2618
|
name = "webcortex-core"
|
|
2619
|
-
version = "2.
|
|
2619
|
+
version = "2.2.0"
|
|
2620
2620
|
dependencies = [
|
|
2621
2621
|
"base64 0.23.1",
|
|
2622
2622
|
"bytes",
|
|
@@ -2643,7 +2643,7 @@ dependencies = [
|
|
|
2643
2643
|
|
|
2644
2644
|
[[package]]
|
|
2645
2645
|
name = "webcortex-py"
|
|
2646
|
-
version = "2.
|
|
2646
|
+
version = "2.2.0"
|
|
2647
2647
|
dependencies = [
|
|
2648
2648
|
"bytes",
|
|
2649
2649
|
"futures",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: web-cortex-framework
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
@@ -349,6 +349,49 @@ invoke.
|
|
|
349
349
|
**A full audit trail**, including refused calls, handoffs, compactions and
|
|
350
350
|
approval decisions, at `GET /_webcortex/audit`.
|
|
351
351
|
|
|
352
|
+
## Agents that see, machines that move
|
|
353
|
+
|
|
354
|
+
WebCortex embeds no vision model — it carries pixels to the model that has one,
|
|
355
|
+
and puts the guard rails a physical system needs around the tools that move it.
|
|
356
|
+
|
|
357
|
+
```python
|
|
358
|
+
from webcortex import HTTPError, Image, WebCortex
|
|
359
|
+
|
|
360
|
+
app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
|
|
361
|
+
|
|
362
|
+
@app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
|
|
363
|
+
def camera() -> dict:
|
|
364
|
+
frame = cam.read() # an OpenCV / NumPy frame
|
|
365
|
+
return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
|
|
366
|
+
|
|
367
|
+
@app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
|
|
368
|
+
actuator=True, approval="required")
|
|
369
|
+
def move(joint: str, degrees: float) -> dict:
|
|
370
|
+
if not -90 <= degrees <= 90:
|
|
371
|
+
raise HTTPError(422, "out of range") # limits live in code, not prompts
|
|
372
|
+
return arm.move(joint, degrees)
|
|
373
|
+
|
|
374
|
+
app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
- **The agent sees the frame.** A tool that returns a `webcortex.Image` —
|
|
378
|
+
anywhere in its result — hands the model the picture itself, over both wire
|
|
379
|
+
formats (Anthropic and OpenAI-compatible, so local vision models too) and over
|
|
380
|
+
MCP, where Claude Code and other clients receive it as image content. Agent
|
|
381
|
+
endpoints also take `"images": [...]` with the input.
|
|
382
|
+
- **Frames do not become a token leak.** Images are lifted out before
|
|
383
|
+
tool-result bounding, step records keep their size rather than their base64,
|
|
384
|
+
and only the last `max_images` stay in context as pixels.
|
|
385
|
+
- **An emergency stop that cannot be routed around.** `actuator=True` routes
|
|
386
|
+
refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
|
|
387
|
+
flow, MCP, or an approval granted after the stop. `webcortex halt` /
|
|
388
|
+
`webcortex release`, or `POST /_webcortex/halt` and `/release`.
|
|
389
|
+
- **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
|
|
390
|
+
an inspector agent and an operator agent, gated, scoped and booting halted.
|
|
391
|
+
Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
|
|
392
|
+
|
|
393
|
+
See [Vision and robotics](docs/vision-and-robotics.md).
|
|
394
|
+
|
|
352
395
|
## Security defaults
|
|
353
396
|
|
|
354
397
|
Deny-by-default throughout; relaxing something costs a line, tightening it costs
|
|
@@ -432,7 +475,7 @@ tested, the known limits, and what v2 added to the surface.
|
|
|
432
475
|
|
|
433
476
|
## Status
|
|
434
477
|
|
|
435
|
-
v2.
|
|
478
|
+
v2.2.0. Working and tested: the manifest IR, router, native ops (static /
|
|
436
479
|
query / proxy / page / files / flow), the free-threaded Python bridge,
|
|
437
480
|
authentication and scopes, rate limiting, CORS, security headers, graceful
|
|
438
481
|
shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
|
|
@@ -440,8 +483,9 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
|
|
|
440
483
|
compaction, context providers, memory, flows, two providers (Anthropic and
|
|
441
484
|
OpenAI-compatible, which covers local models), prompt caching, the spend
|
|
442
485
|
ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
|
|
443
|
-
context pack and `evolve
|
|
444
|
-
|
|
486
|
+
context pack and `evolve`, images in and out of agent runs and MCP, actuators
|
|
487
|
+
and the emergency stop. **403 tests** (124 Rust, 279 Python, including a
|
|
488
|
+
54-test adversarial suite and offline end-to-end suites that drive the whole
|
|
445
489
|
agent stack over HTTP), clippy clean.
|
|
446
490
|
|
|
447
491
|
Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
|
|
@@ -321,6 +321,49 @@ invoke.
|
|
|
321
321
|
**A full audit trail**, including refused calls, handoffs, compactions and
|
|
322
322
|
approval decisions, at `GET /_webcortex/audit`.
|
|
323
323
|
|
|
324
|
+
## Agents that see, machines that move
|
|
325
|
+
|
|
326
|
+
WebCortex embeds no vision model — it carries pixels to the model that has one,
|
|
327
|
+
and puts the guard rails a physical system needs around the tools that move it.
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
from webcortex import HTTPError, Image, WebCortex
|
|
331
|
+
|
|
332
|
+
app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
|
|
333
|
+
|
|
334
|
+
@app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
|
|
335
|
+
def camera() -> dict:
|
|
336
|
+
frame = cam.read() # an OpenCV / NumPy frame
|
|
337
|
+
return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
|
|
338
|
+
|
|
339
|
+
@app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
|
|
340
|
+
actuator=True, approval="required")
|
|
341
|
+
def move(joint: str, degrees: float) -> dict:
|
|
342
|
+
if not -90 <= degrees <= 90:
|
|
343
|
+
raise HTTPError(422, "out of range") # limits live in code, not prompts
|
|
344
|
+
return arm.move(joint, degrees)
|
|
345
|
+
|
|
346
|
+
app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
- **The agent sees the frame.** A tool that returns a `webcortex.Image` —
|
|
350
|
+
anywhere in its result — hands the model the picture itself, over both wire
|
|
351
|
+
formats (Anthropic and OpenAI-compatible, so local vision models too) and over
|
|
352
|
+
MCP, where Claude Code and other clients receive it as image content. Agent
|
|
353
|
+
endpoints also take `"images": [...]` with the input.
|
|
354
|
+
- **Frames do not become a token leak.** Images are lifted out before
|
|
355
|
+
tool-result bounding, step records keep their size rather than their base64,
|
|
356
|
+
and only the last `max_images` stay in context as pixels.
|
|
357
|
+
- **An emergency stop that cannot be routed around.** `actuator=True` routes
|
|
358
|
+
refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
|
|
359
|
+
flow, MCP, or an approval granted after the stop. `webcortex halt` /
|
|
360
|
+
`webcortex release`, or `POST /_webcortex/halt` and `/release`.
|
|
361
|
+
- **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
|
|
362
|
+
an inspector agent and an operator agent, gated, scoped and booting halted.
|
|
363
|
+
Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
|
|
364
|
+
|
|
365
|
+
See [Vision and robotics](docs/vision-and-robotics.md).
|
|
366
|
+
|
|
324
367
|
## Security defaults
|
|
325
368
|
|
|
326
369
|
Deny-by-default throughout; relaxing something costs a line, tightening it costs
|
|
@@ -404,7 +447,7 @@ tested, the known limits, and what v2 added to the surface.
|
|
|
404
447
|
|
|
405
448
|
## Status
|
|
406
449
|
|
|
407
|
-
v2.
|
|
450
|
+
v2.2.0. Working and tested: the manifest IR, router, native ops (static /
|
|
408
451
|
query / proxy / page / files / flow), the free-threaded Python bridge,
|
|
409
452
|
authentication and scopes, rate limiting, CORS, security headers, graceful
|
|
410
453
|
shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
|
|
@@ -412,8 +455,9 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
|
|
|
412
455
|
compaction, context providers, memory, flows, two providers (Anthropic and
|
|
413
456
|
OpenAI-compatible, which covers local models), prompt caching, the spend
|
|
414
457
|
ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
|
|
415
|
-
context pack and `evolve
|
|
416
|
-
|
|
458
|
+
context pack and `evolve`, images in and out of agent runs and MCP, actuators
|
|
459
|
+
and the emergency stop. **403 tests** (124 Rust, 279 Python, including a
|
|
460
|
+
54-test adversarial suite and offline end-to-end suites that drive the whole
|
|
417
461
|
agent stack over HTTP), clippy clean.
|
|
418
462
|
|
|
419
463
|
Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
|
|
@@ -404,33 +404,57 @@ impl ModelProvider for OpenAiCompatProvider {
|
|
|
404
404
|
|
|
405
405
|
/// Translate one canonical (Anthropic-shaped) message into OpenAI messages.
|
|
406
406
|
/// A user turn holding tool results becomes one `tool` message per result.
|
|
407
|
+
///
|
|
408
|
+
/// OpenAI `tool` messages carry text only, so images a tool returned travel in
|
|
409
|
+
/// one `user` message after the tool messages, labelled with the call they
|
|
410
|
+
/// came from. Images sent with the input become `image_url` parts.
|
|
407
411
|
pub fn to_openai_messages(role: &str, content: &Value) -> Vec<Value> {
|
|
408
412
|
match (role, content) {
|
|
409
413
|
(_, Value::String(s)) => vec![json!({"role": role, "content": s})],
|
|
410
414
|
("user", Value::Array(blocks)) => {
|
|
411
415
|
let mut out = Vec::new();
|
|
412
416
|
let mut text = String::new();
|
|
417
|
+
let mut parts: Vec<Value> = Vec::new();
|
|
413
418
|
for b in blocks {
|
|
414
419
|
match b.get("type").and_then(|t| t.as_str()) {
|
|
415
|
-
Some("tool_result") =>
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
Some(Value::
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
420
|
+
Some("tool_result") => {
|
|
421
|
+
let id = b.get("tool_use_id").cloned().unwrap_or(Value::Null);
|
|
422
|
+
let (body, images) = match b.get("content") {
|
|
423
|
+
Some(Value::String(s)) => (s.clone(), Vec::new()),
|
|
424
|
+
Some(Value::Array(inner)) => (
|
|
425
|
+
crate::agent::message_text(&Value::Array(inner.clone())),
|
|
426
|
+
inner.iter().filter_map(super::vision::openai_part).collect(),
|
|
427
|
+
),
|
|
428
|
+
Some(other) => (other.to_string(), Vec::new()),
|
|
429
|
+
None => (String::new(), Vec::new()),
|
|
430
|
+
};
|
|
431
|
+
if !images.is_empty() {
|
|
432
|
+
parts.push(json!({"type": "text", "text": format!(
|
|
433
|
+
"Images returned by tool call {}:",
|
|
434
|
+
id.as_str().unwrap_or("?")
|
|
435
|
+
)}));
|
|
436
|
+
parts.extend(images);
|
|
437
|
+
}
|
|
438
|
+
out.push(json!({"role": "tool", "tool_call_id": id, "content": body}));
|
|
439
|
+
}
|
|
424
440
|
Some("text") => {
|
|
425
441
|
if let Some(t) = b.get("text").and_then(|t| t.as_str()) {
|
|
426
442
|
text.push_str(t);
|
|
427
443
|
}
|
|
428
444
|
}
|
|
445
|
+
Some("image") => parts.extend(super::vision::openai_part(b)),
|
|
429
446
|
_ => {}
|
|
430
447
|
}
|
|
431
448
|
}
|
|
432
|
-
if
|
|
433
|
-
|
|
449
|
+
if parts.is_empty() {
|
|
450
|
+
if !text.is_empty() {
|
|
451
|
+
out.push(json!({"role": "user", "content": text}));
|
|
452
|
+
}
|
|
453
|
+
} else {
|
|
454
|
+
if !text.is_empty() {
|
|
455
|
+
parts.push(json!({"type": "text", "text": text}));
|
|
456
|
+
}
|
|
457
|
+
out.push(json!({"role": "user", "content": parts}));
|
|
434
458
|
}
|
|
435
459
|
out
|
|
436
460
|
}
|
|
@@ -878,10 +902,20 @@ impl ModelProvider for FakeProvider {
|
|
|
878
902
|
};
|
|
879
903
|
|
|
880
904
|
let last = req.conversation.messages.last();
|
|
905
|
+
let is_tool_results = |c: &Value| {
|
|
906
|
+
c.as_array().is_some_and(|bs| {
|
|
907
|
+
bs.iter().any(|b| b.get("type").and_then(|t| t.as_str()) == Some("tool_result"))
|
|
908
|
+
})
|
|
909
|
+
};
|
|
881
910
|
let last_text = match last.map(|m| &m.content) {
|
|
882
911
|
Some(Value::String(s)) => Some(s.clone()),
|
|
912
|
+
// Input sent with images: the text block is the prompt.
|
|
913
|
+
Some(c @ Value::Array(_)) if !is_tool_results(c) => Some(crate::agent::message_text(c)),
|
|
883
914
|
_ => None,
|
|
884
915
|
};
|
|
916
|
+
// So a test can see that pixels reached the model.
|
|
917
|
+
let seen = last.map(|m| super::vision::count(&m.content)).unwrap_or(0);
|
|
918
|
+
let images_note = if seen > 0 { format!(" [saw {seen} image(s)]") } else { String::new() };
|
|
885
919
|
|
|
886
920
|
// A forced tool: structured output.
|
|
887
921
|
if let Some(forced) = req.force_tool {
|
|
@@ -914,7 +948,8 @@ impl ModelProvider for FakeProvider {
|
|
|
914
948
|
let summary = match last.map(|m| &m.content) {
|
|
915
949
|
Some(Value::Array(blocks)) => blocks
|
|
916
950
|
.iter()
|
|
917
|
-
.filter_map(|b| b.get("content")
|
|
951
|
+
.filter_map(|b| b.get("content"))
|
|
952
|
+
.map(|c| c.as_str().map(str::to_string).unwrap_or_else(|| crate::agent::message_text(c)))
|
|
918
953
|
.map(|s| {
|
|
919
954
|
let n = s.chars().count();
|
|
920
955
|
let tail: String = s.chars().skip(n.saturating_sub(160)).collect();
|
|
@@ -925,7 +960,7 @@ impl ModelProvider for FakeProvider {
|
|
|
925
960
|
_ => String::new(),
|
|
926
961
|
};
|
|
927
962
|
|
|
928
|
-
let reply = format!("{} says: {summary}", req.agent.name);
|
|
963
|
+
let reply = format!("{} says: {summary}{images_note}", req.agent.name);
|
|
929
964
|
return done(
|
|
930
965
|
reply.clone(),
|
|
931
966
|
Vec::new(),
|
|
@@ -964,7 +999,7 @@ impl ModelProvider for FakeProvider {
|
|
|
964
999
|
return done(String::new(), calls, Value::Array(raw), StopReason::ToolUse);
|
|
965
1000
|
}
|
|
966
1001
|
|
|
967
|
-
let reply = format!("{} echoes: {text}", req.agent.name);
|
|
1002
|
+
let reply = format!("{} echoes: {text}{images_note}", req.agent.name);
|
|
968
1003
|
done(
|
|
969
1004
|
reply.clone(),
|
|
970
1005
|
Vec::new(),
|
|
@@ -1105,6 +1140,27 @@ mod tests {
|
|
|
1105
1140
|
assert_eq!(out[1]["content"], "{\"n\":2}");
|
|
1106
1141
|
}
|
|
1107
1142
|
|
|
1143
|
+
#[test]
|
|
1144
|
+
fn images_reach_openai_as_image_url_parts_after_the_tool_messages() {
|
|
1145
|
+
let img = json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}});
|
|
1146
|
+
let content = json!([
|
|
1147
|
+
{"type": "tool_result", "tool_use_id": "c1", "is_error": false,
|
|
1148
|
+
"content": [{"type": "text", "text": "{\"frame\":\"[image 1]\"}"}, img]},
|
|
1149
|
+
]);
|
|
1150
|
+
let out = to_openai_messages("user", &content);
|
|
1151
|
+
assert_eq!(out.len(), 2, "tool message, then the images");
|
|
1152
|
+
assert_eq!(out[0]["role"], "tool");
|
|
1153
|
+
assert_eq!(out[0]["content"], "{\"frame\":\"[image 1]\"}");
|
|
1154
|
+
assert_eq!(out[1]["role"], "user");
|
|
1155
|
+
assert_eq!(out[1]["content"][1]["image_url"]["url"], "data:image/png;base64,AAAA");
|
|
1156
|
+
|
|
1157
|
+
let input = json!([img, {"type": "text", "text": "what is this?"}]);
|
|
1158
|
+
let out = to_openai_messages("user", &input);
|
|
1159
|
+
assert_eq!(out.len(), 1);
|
|
1160
|
+
assert_eq!(out[0]["content"][0]["type"], "image_url");
|
|
1161
|
+
assert_eq!(out[0]["content"][1]["text"], "what is this?");
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1108
1164
|
#[test]
|
|
1109
1165
|
fn canonical_assistant_tool_use_becomes_openai_tool_calls() {
|
|
1110
1166
|
let content = json!([
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
//! Images in and out of a run.
|
|
2
|
+
//!
|
|
3
|
+
//! WebCortex does not run vision models — that would be embedded inference.
|
|
4
|
+
//! It carries pixels to the model that does: an image given with the request,
|
|
5
|
+
//! or a frame a tool returned, becomes an image block in the conversation.
|
|
6
|
+
//!
|
|
7
|
+
//! A tool marks an image in its JSON result with a one-key object,
|
|
8
|
+
//! `{"$image": {"media_type": "image/png", "data": "<base64>"}}` or
|
|
9
|
+
//! `{"$image": {"url": "https://…"}}` — which is what `webcortex.Image`
|
|
10
|
+
//! serialises to. The runtime lifts each one out of the result, leaves a short
|
|
11
|
+
//! placeholder in its place, and attaches the image beside the text, so the
|
|
12
|
+
//! model sees the picture and the step record keeps only its size.
|
|
13
|
+
//!
|
|
14
|
+
//! Images are expensive context: every step replays the conversation, and a
|
|
15
|
+
//! camera tool called in a loop would resend every frame it ever took.
|
|
16
|
+
//! [`prune`] keeps the most recent few and turns the rest into a line of text.
|
|
17
|
+
|
|
18
|
+
use base64::Engine;
|
|
19
|
+
use serde_json::{Value, json};
|
|
20
|
+
|
|
21
|
+
/// The key that marks an image inside a tool result or a request.
|
|
22
|
+
pub const IMAGE_KEY: &str = "$image";
|
|
23
|
+
|
|
24
|
+
/// Media types every supported provider accepts.
|
|
25
|
+
pub const MEDIA_TYPES: [&str; 4] = ["image/png", "image/jpeg", "image/gif", "image/webp"];
|
|
26
|
+
|
|
27
|
+
/// Decoded size ceiling for one image. Providers reject larger ones anyway;
|
|
28
|
+
/// refusing here gives a clearer error and keeps them out of session memory.
|
|
29
|
+
pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
|
|
30
|
+
|
|
31
|
+
/// Images accepted with one request, and lifted out of one tool result.
|
|
32
|
+
pub const MAX_IMAGES: usize = 16;
|
|
33
|
+
|
|
34
|
+
/// Turn one image spec into a canonical (Anthropic-shaped) image block.
|
|
35
|
+
///
|
|
36
|
+
/// Accepts `{"media_type", "data"}`, `{"url"}`, or either wrapped in
|
|
37
|
+
/// `{"$image": …}`.
|
|
38
|
+
pub fn image_block(spec: &Value) -> Result<Value, String> {
|
|
39
|
+
let spec = spec.get(IMAGE_KEY).unwrap_or(spec);
|
|
40
|
+
if let Some(url) = spec.get("url").and_then(|u| u.as_str()) {
|
|
41
|
+
if !(url.starts_with("https://") || url.starts_with("http://")) {
|
|
42
|
+
return Err(format!("image url must be http(s): {url:?}"));
|
|
43
|
+
}
|
|
44
|
+
return Ok(json!({"type": "image", "source": {"type": "url", "url": url}}));
|
|
45
|
+
}
|
|
46
|
+
let data = spec
|
|
47
|
+
.get("data")
|
|
48
|
+
.and_then(|d| d.as_str())
|
|
49
|
+
.ok_or("an image needs base64 'data' with a 'media_type', or a 'url'")?;
|
|
50
|
+
let media_type = spec
|
|
51
|
+
.get("media_type")
|
|
52
|
+
.and_then(|m| m.as_str())
|
|
53
|
+
.ok_or("an image with 'data' needs a 'media_type'")?;
|
|
54
|
+
if !MEDIA_TYPES.contains(&media_type) {
|
|
55
|
+
return Err(format!(
|
|
56
|
+
"unsupported image media_type {media_type:?}; use one of {}",
|
|
57
|
+
MEDIA_TYPES.join(", ")
|
|
58
|
+
));
|
|
59
|
+
}
|
|
60
|
+
let bytes = base64::engine::general_purpose::STANDARD
|
|
61
|
+
.decode(data)
|
|
62
|
+
.map_err(|e| format!("image data is not valid base64: {e}"))?;
|
|
63
|
+
if bytes.is_empty() {
|
|
64
|
+
return Err("image data is empty".into());
|
|
65
|
+
}
|
|
66
|
+
if bytes.len() > MAX_IMAGE_BYTES {
|
|
67
|
+
return Err(format!(
|
|
68
|
+
"image is {} bytes; the limit is {MAX_IMAGE_BYTES}",
|
|
69
|
+
bytes.len()
|
|
70
|
+
));
|
|
71
|
+
}
|
|
72
|
+
Ok(json!({
|
|
73
|
+
"type": "image",
|
|
74
|
+
"source": {"type": "base64", "media_type": media_type, "data": data},
|
|
75
|
+
}))
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/// The `images` field of an agent request, as canonical blocks.
|
|
79
|
+
pub fn input_images(value: Option<&Value>) -> Result<Vec<Value>, String> {
|
|
80
|
+
let Some(value) = value else { return Ok(Vec::new()) };
|
|
81
|
+
if value.is_null() {
|
|
82
|
+
return Ok(Vec::new());
|
|
83
|
+
}
|
|
84
|
+
let items = value.as_array().ok_or("'images' must be a list")?;
|
|
85
|
+
if items.len() > MAX_IMAGES {
|
|
86
|
+
return Err(format!("at most {MAX_IMAGES} images per request ({} given)", items.len()));
|
|
87
|
+
}
|
|
88
|
+
items
|
|
89
|
+
.iter()
|
|
90
|
+
.enumerate()
|
|
91
|
+
.map(|(i, v)| image_block(v).map_err(|e| format!("images[{i}]: {e}")))
|
|
92
|
+
.collect()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// True for a `{"$image": …}` marker object.
|
|
96
|
+
fn is_marker(value: &Value) -> bool {
|
|
97
|
+
value.as_object().is_some_and(|o| o.len() == 1 && o.contains_key(IMAGE_KEY))
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Decoded size of a base64 string, without decoding it.
|
|
101
|
+
fn decoded_len(data: &str) -> usize {
|
|
102
|
+
let pad = data.bytes().rev().take_while(|&b| b == b'=').count();
|
|
103
|
+
(data.len() / 4 * 3).saturating_sub(pad)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/// A one-line description of an image block, for placeholders and transcripts.
|
|
107
|
+
pub fn describe(block: &Value) -> String {
|
|
108
|
+
let source = block.get("source").cloned().unwrap_or(Value::Null);
|
|
109
|
+
match source.get("type").and_then(|t| t.as_str()) {
|
|
110
|
+
Some("url") => format!("image at {}", source.get("url").and_then(|u| u.as_str()).unwrap_or("?")),
|
|
111
|
+
_ => format!(
|
|
112
|
+
"{}, {} bytes",
|
|
113
|
+
source.get("media_type").and_then(|m| m.as_str()).unwrap_or("image"),
|
|
114
|
+
decoded_len(source.get("data").and_then(|d| d.as_str()).unwrap_or(""))
|
|
115
|
+
),
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/// Lift every image marker out of a tool result.
|
|
120
|
+
///
|
|
121
|
+
/// Returns the result with each marker replaced by a placeholder string, and
|
|
122
|
+
/// the images as canonical blocks in the order they were found. An invalid
|
|
123
|
+
/// marker, or one past `max`, becomes a placeholder saying why — a tool that
|
|
124
|
+
/// returns a bad frame should not fail the run.
|
|
125
|
+
pub fn extract(value: &Value, max: usize) -> (Value, Vec<Value>) {
|
|
126
|
+
let mut images = Vec::new();
|
|
127
|
+
let text = walk(value, &mut |marker| {
|
|
128
|
+
if images.len() >= max {
|
|
129
|
+
return format!("[image omitted: at most {max} images per tool result]");
|
|
130
|
+
}
|
|
131
|
+
match image_block(marker) {
|
|
132
|
+
Ok(block) => {
|
|
133
|
+
let line = format!("[image {}: {} — attached]", images.len() + 1, describe(&block));
|
|
134
|
+
images.push(block);
|
|
135
|
+
line
|
|
136
|
+
}
|
|
137
|
+
Err(e) => format!("[image omitted: {e}]"),
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
(text, images)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/// The result as a step record or structured output keeps it: every marker's
|
|
144
|
+
/// payload replaced by its size, so megabytes of base64 do not land in the
|
|
145
|
+
/// response body, the audit log or session memory twice.
|
|
146
|
+
pub fn redact(value: &Value) -> Value {
|
|
147
|
+
walk(value, &mut |marker| {
|
|
148
|
+
let spec = marker.get(IMAGE_KEY).unwrap_or(&Value::Null);
|
|
149
|
+
match spec.get("data").and_then(|d| d.as_str()) {
|
|
150
|
+
Some(data) => json!({IMAGE_KEY: {
|
|
151
|
+
"media_type": spec.get("media_type").cloned().unwrap_or(Value::Null),
|
|
152
|
+
"bytes": decoded_len(data),
|
|
153
|
+
}}),
|
|
154
|
+
None => marker.clone(),
|
|
155
|
+
}
|
|
156
|
+
})
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/// True when the value holds at least one image marker.
|
|
160
|
+
pub fn contains_images(value: &Value) -> bool {
|
|
161
|
+
match value {
|
|
162
|
+
v if is_marker(v) => true,
|
|
163
|
+
Value::Array(items) => items.iter().any(contains_images),
|
|
164
|
+
Value::Object(map) => map.values().any(contains_images),
|
|
165
|
+
_ => false,
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
fn walk<T: Into<Value>>(value: &Value, on_marker: &mut impl FnMut(&Value) -> T) -> Value {
|
|
170
|
+
match value {
|
|
171
|
+
v if is_marker(v) => on_marker(v).into(),
|
|
172
|
+
Value::Array(items) => Value::Array(items.iter().map(|v| walk(v, on_marker)).collect()),
|
|
173
|
+
Value::Object(map) => Value::Object(
|
|
174
|
+
map.iter().map(|(k, v)| (k.clone(), walk(v, on_marker))).collect(),
|
|
175
|
+
),
|
|
176
|
+
other => other.clone(),
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/// Keep only the `keep` most recent images in a conversation; older ones
|
|
181
|
+
/// become a line of text. Returns how many were dropped.
|
|
182
|
+
///
|
|
183
|
+
/// Message structure is untouched — a `tool_result` keeps its id and stays
|
|
184
|
+
/// paired with its `tool_use` — only image blocks are swapped for text.
|
|
185
|
+
pub fn prune(messages: &mut [crate::agent::Message], keep: usize) -> usize {
|
|
186
|
+
let total: usize = messages.iter().map(|m| count(&m.content)).sum();
|
|
187
|
+
if total <= keep {
|
|
188
|
+
return 0;
|
|
189
|
+
}
|
|
190
|
+
let mut to_drop = total - keep;
|
|
191
|
+
let dropped = to_drop;
|
|
192
|
+
for m in messages.iter_mut() {
|
|
193
|
+
if to_drop == 0 {
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
drop_oldest(&mut m.content, &mut to_drop);
|
|
197
|
+
}
|
|
198
|
+
dropped
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/// Image blocks in one message's content, including inside tool results.
|
|
202
|
+
pub fn count(content: &Value) -> usize {
|
|
203
|
+
match content {
|
|
204
|
+
Value::Array(blocks) => blocks
|
|
205
|
+
.iter()
|
|
206
|
+
.map(|b| match b.get("type").and_then(|t| t.as_str()) {
|
|
207
|
+
Some("image") => 1,
|
|
208
|
+
Some("tool_result") => b.get("content").map(count).unwrap_or(0),
|
|
209
|
+
_ => 0,
|
|
210
|
+
})
|
|
211
|
+
.sum(),
|
|
212
|
+
_ => 0,
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
fn drop_oldest(content: &mut Value, to_drop: &mut usize) {
|
|
217
|
+
let Value::Array(blocks) = content else { return };
|
|
218
|
+
for b in blocks.iter_mut() {
|
|
219
|
+
if *to_drop == 0 {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
match b.get("type").and_then(|t| t.as_str()) {
|
|
223
|
+
Some("image") => {
|
|
224
|
+
*b = json!({"type": "text", "text": format!(
|
|
225
|
+
"[earlier image ({}) dropped from context to save tokens]",
|
|
226
|
+
describe(b)
|
|
227
|
+
)});
|
|
228
|
+
*to_drop -= 1;
|
|
229
|
+
}
|
|
230
|
+
Some("tool_result") => {
|
|
231
|
+
if let Some(inner) = b.get_mut("content") {
|
|
232
|
+
drop_oldest(inner, to_drop);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
_ => {}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/// A canonical image block as an OpenAI `image_url` content part.
|
|
241
|
+
pub fn openai_part(block: &Value) -> Option<Value> {
|
|
242
|
+
let source = block.get("source")?;
|
|
243
|
+
let url = match source.get("type").and_then(|t| t.as_str()) {
|
|
244
|
+
Some("url") => source.get("url")?.as_str()?.to_string(),
|
|
245
|
+
Some("base64") => format!(
|
|
246
|
+
"data:{};base64,{}",
|
|
247
|
+
source.get("media_type")?.as_str()?,
|
|
248
|
+
source.get("data")?.as_str()?
|
|
249
|
+
),
|
|
250
|
+
_ => return None,
|
|
251
|
+
};
|
|
252
|
+
Some(json!({"type": "image_url", "image_url": {"url": url}}))
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
#[cfg(test)]
|
|
256
|
+
mod tests {
|
|
257
|
+
use super::*;
|
|
258
|
+
use crate::agent::Message;
|
|
259
|
+
|
|
260
|
+
// A 1x1 transparent PNG.
|
|
261
|
+
const PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
|
|
262
|
+
|
|
263
|
+
fn marker() -> Value {
|
|
264
|
+
json!({"$image": {"media_type": "image/png", "data": PNG}})
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
#[test]
|
|
268
|
+
fn a_marker_becomes_a_canonical_block() {
|
|
269
|
+
let block = image_block(&marker()).unwrap();
|
|
270
|
+
assert_eq!(block["type"], "image");
|
|
271
|
+
assert_eq!(block["source"]["type"], "base64");
|
|
272
|
+
assert_eq!(block["source"]["media_type"], "image/png");
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
#[test]
|
|
276
|
+
fn bad_images_are_refused_with_a_reason() {
|
|
277
|
+
assert!(image_block(&json!({"media_type": "image/bmp", "data": PNG})).unwrap_err().contains("unsupported"));
|
|
278
|
+
assert!(image_block(&json!({"media_type": "image/png", "data": "!!"})).unwrap_err().contains("base64"));
|
|
279
|
+
assert!(image_block(&json!({"url": "file:///etc/passwd"})).unwrap_err().contains("http"));
|
|
280
|
+
assert!(image_block(&json!({"data": PNG})).unwrap_err().contains("media_type"));
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
#[test]
|
|
284
|
+
fn urls_pass_through() {
|
|
285
|
+
let block = image_block(&json!({"url": "https://example.com/a.jpg"})).unwrap();
|
|
286
|
+
assert_eq!(block["source"]["url"], "https://example.com/a.jpg");
|
|
287
|
+
assert_eq!(openai_part(&block).unwrap()["image_url"]["url"], "https://example.com/a.jpg");
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
#[test]
|
|
291
|
+
fn extract_lifts_nested_markers_and_leaves_placeholders() {
|
|
292
|
+
let result = json!({"frame": marker(), "objects": [{"label": "bolt", "crop": marker()}], "n": 1});
|
|
293
|
+
let (text, images) = extract(&result, 8);
|
|
294
|
+
assert_eq!(images.len(), 2);
|
|
295
|
+
assert!(text["frame"].as_str().unwrap().starts_with("[image 1: image/png"));
|
|
296
|
+
assert!(text["objects"][0]["crop"].as_str().unwrap().starts_with("[image 2:"));
|
|
297
|
+
assert_eq!(text["n"], 1);
|
|
298
|
+
assert!(!text.to_string().contains(PNG));
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
#[test]
|
|
302
|
+
fn extract_respects_its_limit_and_survives_a_bad_marker() {
|
|
303
|
+
let result = json!([marker(), {"$image": {"media_type": "image/png", "data": "%%"}}, marker(), marker()]);
|
|
304
|
+
let (text, images) = extract(&result, 2);
|
|
305
|
+
assert_eq!(images.len(), 2);
|
|
306
|
+
assert!(text[1].as_str().unwrap().contains("not valid base64"));
|
|
307
|
+
assert!(text[2].as_str().unwrap().starts_with("[image 2:"));
|
|
308
|
+
assert!(text[3].as_str().unwrap().contains("at most 2"));
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
#[test]
|
|
312
|
+
fn redact_keeps_the_size_not_the_pixels() {
|
|
313
|
+
let r = redact(&json!({"frame": marker()}));
|
|
314
|
+
assert_eq!(r["frame"]["$image"]["media_type"], "image/png");
|
|
315
|
+
assert!(r["frame"]["$image"]["bytes"].as_u64().unwrap() > 0);
|
|
316
|
+
assert!(r["frame"]["$image"].get("data").is_none());
|
|
317
|
+
assert!(contains_images(&json!({"a": [marker()]})));
|
|
318
|
+
assert!(!contains_images(&json!({"a": [1, "x"]})));
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
#[test]
|
|
322
|
+
fn prune_drops_the_oldest_images_first() {
|
|
323
|
+
let img = image_block(&marker()).unwrap();
|
|
324
|
+
let mut messages = vec![
|
|
325
|
+
Message { role: "user".into(), content: json!([img.clone(), {"type": "text", "text": "look"}]) },
|
|
326
|
+
Message { role: "assistant".into(), content: json!([{"type": "tool_use", "id": "t1", "name": "cam", "input": {}}]) },
|
|
327
|
+
Message { role: "user".into(), content: json!([{"type": "tool_result", "tool_use_id": "t1",
|
|
328
|
+
"content": [{"type": "text", "text": "frame"}, img.clone()], "is_error": false}]) },
|
|
329
|
+
Message { role: "user".into(), content: json!([img.clone()]) },
|
|
330
|
+
];
|
|
331
|
+
assert_eq!(prune(&mut messages, 1), 2);
|
|
332
|
+
assert_eq!(messages.iter().map(|m| count(&m.content)).sum::<usize>(), 1);
|
|
333
|
+
assert!(messages[0].content[0]["text"].as_str().unwrap().contains("dropped from context"));
|
|
334
|
+
assert_eq!(messages[2].content[0]["tool_use_id"], "t1", "pairing kept");
|
|
335
|
+
assert_eq!(count(&messages[3].content), 1, "the newest survives");
|
|
336
|
+
assert_eq!(prune(&mut messages, 1), 0);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
#[test]
|
|
340
|
+
fn input_images_are_validated_and_capped() {
|
|
341
|
+
assert!(input_images(None).unwrap().is_empty());
|
|
342
|
+
assert_eq!(input_images(Some(&json!([marker(), {"url": "https://x.test/a.png"}]))).unwrap().len(), 2);
|
|
343
|
+
assert!(input_images(Some(&json!("nope"))).is_err());
|
|
344
|
+
let many = Value::Array(vec![marker(); MAX_IMAGES + 1]);
|
|
345
|
+
assert!(input_images(Some(&many)).is_err());
|
|
346
|
+
assert!(input_images(Some(&json!([{"url": "ftp://x"}]))).unwrap_err().starts_with("images[0]"));
|
|
347
|
+
}
|
|
348
|
+
}
|