agentenv-framework-protocol 0.1.381__tar.gz → 0.1.386__tar.gz

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentenv-framework-protocol
-Version: 0.1.381
+Version: 0.1.386
 Summary: Open data-plane protocol and server SDK for agent environments.
 Project-URL: Homepage, https://www.agentenvframework.com
 Project-URL: Documentation, https://www.agentenvframework.com/docs
@@ -35,11 +35,13 @@ Requires-Dist: python-multipart>=0.0.13; extra == 'agent'
 Requires-Dist: regex>=2023.12.25; extra == 'agent'
 Requires-Dist: uvicorn>=0.30; extra == 'agent'
 Provides-Extra: computer
+Requires-Dist: av>=17; extra == 'computer'
 Requires-Dist: mcp<2,>=1.25; extra == 'computer'
 Requires-Dist: pillow>=10.4; extra == 'computer'
 Requires-Dist: uvicorn>=0.30; extra == 'computer'
 Provides-Extra: dev
 Requires-Dist: a2a-sdk[http-server]<0.4,>=0.3.26; extra == 'dev'
+Requires-Dist: av>=17; extra == 'dev'
 Requires-Dist: fastmcp<4,>=3.4; extra == 'dev'
 Requires-Dist: mcp<2,>=1.25; extra == 'dev'
 Requires-Dist: pillow>=10.4; extra == 'dev'
@@ -367,7 +369,7 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session:
 
 A computer driver is what runs inside a computer an agent uses through screenshots and input. Write
 one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, which needs the
-`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow).
+`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn, Pillow and PyAV).
 The driver marks the methods that capture the screen and send input; the base class serves the
 computer contract from them, so the tools, results and errors can't drift from it. A complete
 driver for an X11 desktop is agent-env's reference driver, in its
@@ -418,6 +420,7 @@ pixels), at the display's size times its scale. Input operations may return an `
 | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools |
 | `@run_in_session` | no; a Linux default | `computer-exec/v1` |
 | `@write_files` | no; a Linux default | `data/add` |
+| `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` |
 
 The class is checked when it is defined: a missing required operation, half a group, two methods
 for one operation that some platform could both match, or features that don't match the
@@ -449,8 +452,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
 - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a
   capture, plus the driver's own `ready_checks()`; tools refuse `device_unavailable` until they
   pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool
-  descriptions), `computer-keyboard/v1` and `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?,
-  request_id}`: output capped per stream, a repeated `request_id` answered with the first result).
+  descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?,
+  request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and,
+  with `recording`, `computer-recording/v1` (below).
 - **The data plane.** `data/add` takes file parts that name their path, relative to the session
   user's home, under the `data-objects/v1` marker (`{"path": ...}`), as inline bytes, an https URL,
   or a `transfers.ReadObject` read grant under the marker's `read`, and writes them atomically.
@@ -476,6 +480,47 @@ names the caller whose latest frame and others' input a call uses.
 `MyDriver.declaration(platform)` returns the card the class serves on a `Platform`, built without
 running the driver, its operations or its dependencies.
 
+### Recording the display
+
+Every driver serves `computer-recording/v1`: video of the display, as MP4 (H.264) segments, for the
+framework to start and end around the work it wants on record. `recording = Recording(...)` sets its
+limits, and `recording = None` turns it off.
+
+```python
+from agentenv_protocol.computer import RecordedFrame, Recording, record
+
+
+class MyDriver(ComputerDriverEnvironment):
+    recording = Recording(max_fps=30)      # max_concurrent=2, max_duration_s=86400, max_segment_s=15
+
+    @record
+    async def stream(self, hints):         # hints.scope, fps, max_edge, cursor, max_segment_s, directory
+        async for image, captured_at in my_screen_stream(hints.fps):
+            yield RecordedFrame(image, captured_at)   # a Pillow image or a PyAV VideoFrame
+```
+
+- **The stream.** `@record` yields `RecordedFrame`s from a stream the OS pushes, which the base class
+  scales, encodes and splits into segments, each frame timed by its `captured_at`; or
+  `RecordedSegment`s, files a native recorder finished under `hints.directory`, which it only hashes,
+  uploads and lists. The base class closes the stream at `end`. A source that blocks, as most OS
+  capture APIs do, goes through `frames_in_thread(frames)`: the generator `frames()` runs in a thread
+  of its own, the latest 8 frames wait for the encoder, and it is closed when the recording stops. A
+  driver without `@record` records by the base class polling `@capture`, with a pointer drawn at
+  `@pointer_position`: the slow path, at most 15 fps.
+- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?}` returns
+  `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment
+  and returns the manifest. A repeated `request_id`, or a repeated `end`, answers the same. A start
+  past `max_concurrent` is refused `limit_exceeded`; `max_duration_s`, free disk under 512 MB or a stream
+  failing for 60 s end a recording by itself.
+- **Segments.** A segment closes at `max_segment_s`, when the frame size changes or after a gap in the
+  stream. Each uploads as it closes under `start`'s `write_namespace` grant, at
+  `<recording_id>/<index>.mp4`, and is then deleted; `end`'s own grant uploads any that missed. With no
+  grant they stay under `~/.agentenv-recordings/`.
+- **The manifest** (`RecordingManifest`). Its `segments` (`ManifestSegment`) carry each one's path,
+  `started_at` and `ended_at` (its first and last frames' capture times), size, frame count, byte size,
+  SHA-256 and whether it uploaded, and `ended_by` says why the recording ended: `end`, `reset`,
+  `teardown`, `limit` or `error`. A field it doesn't know is left out.
+
 ### Checking a driver: the conformance probe
 
 The probe checks a running driver against the contract through its card, tools and extensions. It
@@ -498,7 +543,7 @@ The CLI exits 0 when no check failed, 1 when one did and 2 when it can't reach t
 | Level | Checks | Needs on the computer |
 |---|---|---|
 | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing |
-| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add` | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` |
+| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` |
 
 At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into
 `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/README.md

@@ -316,7 +316,7 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session:
 
 A computer driver is what runs inside a computer an agent uses through screenshots and input. Write
 one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, which needs the
-`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow).
+`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn, Pillow and PyAV).
 The driver marks the methods that capture the screen and send input; the base class serves the
 computer contract from them, so the tools, results and errors can't drift from it. A complete
 driver for an X11 desktop is agent-env's reference driver, in its
@@ -367,6 +367,7 @@ pixels), at the display's size times its scale. Input operations may return an `
 | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools |
 | `@run_in_session` | no; a Linux default | `computer-exec/v1` |
 | `@write_files` | no; a Linux default | `data/add` |
+| `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` |
 
 The class is checked when it is defined: a missing required operation, half a group, two methods
 for one operation that some platform could both match, or features that don't match the
@@ -398,8 +399,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
 - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a
   capture, plus the driver's own `ready_checks()`; tools refuse `device_unavailable` until they
   pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool
-  descriptions), `computer-keyboard/v1` and `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?,
-  request_id}`: output capped per stream, a repeated `request_id` answered with the first result).
+  descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?,
+  request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and,
+  with `recording`, `computer-recording/v1` (below).
 - **The data plane.** `data/add` takes file parts that name their path, relative to the session
   user's home, under the `data-objects/v1` marker (`{"path": ...}`), as inline bytes, an https URL,
   or a `transfers.ReadObject` read grant under the marker's `read`, and writes them atomically.
@@ -425,6 +427,47 @@ names the caller whose latest frame and others' input a call uses.
 `MyDriver.declaration(platform)` returns the card the class serves on a `Platform`, built without
 running the driver, its operations or its dependencies.
 
+### Recording the display
+
+Every driver serves `computer-recording/v1`: video of the display, as MP4 (H.264) segments, for the
+framework to start and end around the work it wants on record. `recording = Recording(...)` sets its
+limits, and `recording = None` turns it off.
+
+```python
+from agentenv_protocol.computer import RecordedFrame, Recording, record
+
+
+class MyDriver(ComputerDriverEnvironment):
+    recording = Recording(max_fps=30)      # max_concurrent=2, max_duration_s=86400, max_segment_s=15
+
+    @record
+    async def stream(self, hints):         # hints.scope, fps, max_edge, cursor, max_segment_s, directory
+        async for image, captured_at in my_screen_stream(hints.fps):
+            yield RecordedFrame(image, captured_at)   # a Pillow image or a PyAV VideoFrame
+```
+
+- **The stream.** `@record` yields `RecordedFrame`s from a stream the OS pushes, which the base class
+  scales, encodes and splits into segments, each frame timed by its `captured_at`; or
+  `RecordedSegment`s, files a native recorder finished under `hints.directory`, which it only hashes,
+  uploads and lists. The base class closes the stream at `end`. A source that blocks, as most OS
+  capture APIs do, goes through `frames_in_thread(frames)`: the generator `frames()` runs in a thread
+  of its own, the latest 8 frames wait for the encoder, and it is closed when the recording stops. A
+  driver without `@record` records by the base class polling `@capture`, with a pointer drawn at
+  `@pointer_position`: the slow path, at most 15 fps.
+- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?}` returns
+  `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment
+  and returns the manifest. A repeated `request_id`, or a repeated `end`, answers the same. A start
+  past `max_concurrent` is refused `limit_exceeded`; `max_duration_s`, free disk under 512 MB or a stream
+  failing for 60 s end a recording by itself.
+- **Segments.** A segment closes at `max_segment_s`, when the frame size changes or after a gap in the
+  stream. Each uploads as it closes under `start`'s `write_namespace` grant, at
+  `<recording_id>/<index>.mp4`, and is then deleted; `end`'s own grant uploads any that missed. With no
+  grant they stay under `~/.agentenv-recordings/`.
+- **The manifest** (`RecordingManifest`). Its `segments` (`ManifestSegment`) carry each one's path,
+  `started_at` and `ended_at` (its first and last frames' capture times), size, frame count, byte size,
+  SHA-256 and whether it uploaded, and `ended_by` says why the recording ended: `end`, `reset`,
+  `teardown`, `limit` or `error`. A field it doesn't know is left out.
+
 ### Checking a driver: the conformance probe
 
 The probe checks a running driver against the contract through its card, tools and extensions. It
@@ -447,7 +490,7 @@ The CLI exits 0 when no check failed, 1 when one did and 2 when it can't reach t
 | Level | Checks | Needs on the computer |
 |---|---|---|
 | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing |
-| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add` | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` |
+| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` |
 
 At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into
 `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentenv-framework-protocol"
-version = "0.1.381"
+version = "0.1.386"
 description = "Open data-plane protocol and server SDK for agent environments."
 readme = "README.md"
 license = "Apache-2.0"
@@ -35,8 +35,10 @@ agent = [
     "regex>=2023.12.25",
     "uvicorn>=0.30",
 ]
-# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP and crops and encodes frames.
+# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP, crops and encodes frames, and
+# records the display as H.264 with PyAV (computer-recording/v1).
 computer = [
+    "av>=17",
     "mcp>=1.25,<2",
     "Pillow>=10.4",
     "uvicorn>=0.30",
@@ -50,6 +52,7 @@ dev = [
     "mcp>=1.25,<2",
     "fastmcp>=3.4,<4",  # the standalone-fastmcp mount path
     "Pillow>=10.4",  # the computer extra's frame crops and encodings
+    "av>=17",  # the computer extra's encoder, and the tests' decoder
 ]
 [project.urls]
 Homepage = "https://www.agentenvframework.com"

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/a2a_agent/_triggers.py

@@ -206,6 +206,7 @@ class TriggerEngine:
         solver_message: str = "",
         context_id: str = "default",
         env_triggers: Any = None,
+        record_unfired: bool = True,
     ) -> dict[str, Any]:
         if isinstance(turn, bool) or not isinstance(turn, int) or turn < 1:
             raise TriggerError("decide: 'turn' must be a positive integer")
@@ -256,6 +257,8 @@ class TriggerEngine:
                     elif action["type"] == "end":
                         done = True
 
+        if not fired_ids and not record_unfired:
+            return {"parts": [], "done": False, "fired": []}
         # Commit only after every predicate succeeds. A timed-out decision can then
         # be retried without consuming the turn or partially firing once-only rules.
         context = self._context_state(context_id)

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/a2a_agent/extensions.py

@@ -285,6 +285,7 @@ class TriggerDecideRequest(ExtensionRequest):
     solver_message: str = Field(default="", max_length=MAX_SOLVER_MESSAGE_LENGTH)
     context_id: str = "default"
     env_triggers: dict[str, dict[str, Any]] | None = None
+    record_unfired: bool = True  # False: a decision where nothing fires changes no state, so it can be repeated
 
 
 PROXY_TOKEN_MIN_LENGTH = 22

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/a2a_agent/framework.py

@@ -836,6 +836,7 @@ class _SdkServices:
             solver_message=request.solver_message,
             context_id=request.context_id,
             env_triggers=request.env_triggers,
+            record_unfired=request.record_unfired,
         )
 
     async def triggers_state(self) -> dict[str, Any]:

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/__init__.py

@@ -25,6 +25,7 @@ from .card import (
     COMPUTER_KEYBOARD,
     COMPUTER_POINTER,
     COMPUTER_READY,
+    COMPUTER_RECORDING,
     COMPUTER_SPACES,
 )
 from .operations import (
@@ -44,13 +45,16 @@ from .operations import (
     pointer_position,
     pointer_up,
     press,
+    record,
     run_in_session,
     scroll,
     set_display,
     type_text,
     write_files,
 )
+from .manifest import ManifestSegment, RecordingManifest
 from .platform import Platform
+from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording, frames_in_thread
 from .settings import SETTINGS_ENV, ComputerSettings
 
 __all__ = [
@@ -58,10 +62,11 @@ __all__ = [
     "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV",
     "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press",
     "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up",
-    "run_in_session", "write_files",
+    "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment",
+    "frames_in_thread", "RecordingManifest", "ManifestSegment",
     "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported",
     "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed",
     "Internal",
     "COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD",
-    "COMPUTER_EXEC",
+    "COMPUTER_EXEC", "COMPUTER_RECORDING",
 ]

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/card.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+from .recording import recording_params
 from ..types import METHOD_ADD, EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension
 
 if TYPE_CHECKING:
@@ -17,6 +18,7 @@ COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1"
 COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1"
 COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1"
 COMPUTER_EXEC = "urn:agentenv:computer-exec/v1"
+COMPUTER_RECORDING = "urn:agentenv:computer-recording/v1"
 
 MAX_EXEC_TIMEOUT_S = 3600
 KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)}
@@ -48,6 +50,8 @@ def served_extensions(driver: ComputerDriverEnvironment,
         need = _EXTENSION_NEEDS.get(extension.uri)
         if need is not None and not driver.has(need):
             continue
+        if extension.uri == COMPUTER_RECORDING and driver.recording is None:
+            continue
         dropped = {op for (uri, op), need in _METHOD_NEEDS.items() if uri == extension.uri and not driver.has(need)}
         if dropped:
             methods = {op: entry for op, entry in extension.params["methods"].items() if op not in dropped}
@@ -70,6 +74,8 @@ def extension_params(driver: ComputerDriverEnvironment, uri: str) -> dict:
                 "layouts": list(driver.layouts)}
     if uri == COMPUTER_EXEC:
         return {"max_output_bytes": driver.settings.exec_output_bytes, "max_timeout_s": MAX_EXEC_TIMEOUT_S}
+    if uri == COMPUTER_RECORDING and driver.recording is not None:
+        return recording_params(driver.recording, driver.has("record"))
     return {}
 
 

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/checks.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from ..errors import AgentEnvError
 from .keys import canonical
 from .operations import declared
+from .recording import check_recording
 
 BUTTONS = ("left", "right", "middle", "back", "forward")
 
@@ -38,5 +39,8 @@ def check_features(cls: type) -> None:
             problems.append(f"keys: {e.message}")
     if ("set_display" in declared(cls)) != bool(cls.display_modes):
         problems.append("display_modes and @set_display come together")
+    problems += check_recording(cls.recording)
+    if "record" in declared(cls) and cls.recording is None:
+        problems.append("@record needs recording, which recording = None turns off")
     if problems:
         raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems))

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/conformance/checks.py

@@ -4,15 +4,19 @@ fixture with data/add, launches it with computer-exec and reads its state back t
 from __future__ import annotations
 
 import asyncio
+import base64
+import contextlib
 import json
 import math
 import time
 import uuid
 from collections.abc import Awaitable, Callable
+from datetime import datetime
 from importlib import resources
 
 from mcp import ClientSession
 from mcp.types import CallToolResult
+from pydantic import ValidationError
 
 from ... import client as protocol
 from ...client import ExtensionCallError
@@ -25,11 +29,15 @@ from ..card import (
     COMPUTER_KEYBOARD,
     COMPUTER_POINTER,
     COMPUTER_READY,
+    COMPUTER_RECORDING,
     COMPUTER_SPACES,
     GROUP_TOOLS,
     POINT_TOOLS,
 )
+from ..manifest import RecordingManifest
+from ..recording import RECORDINGS_DIR
 from .client import Driver, image
+from .mp4 import read_mp4
 from .report import CheckResult
 
 MINIMUM_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type",
@@ -45,6 +53,16 @@ POLL_S = 0.15
 #: How long the fixture may take to start.
 FIXTURE_START_S = 30.0
 NORM = 1000
+#: How long the probe records, and the frame rate it asks for.
+RECORD_S = 2.0
+RECORD_FPS = 5
+#: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer.
+READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();"
+               "print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest()}))")
+#: Prints ``length`` bytes of a file from ``offset``, base64, under computer-exec's output cap.
+READ_CHUNK = ("import base64,sys;f=open(sys.argv[1],'rb');f.seek(int(sys.argv[2]));"
+              "print(base64.b64encode(f.read(int(sys.argv[3]))).decode())")
+CHUNK_BYTES = 512 << 10
 #: The checks only ``full`` runs, recorded unverified when the fixture can't be staged.
 FULL_CHECKS = (("P3", "fixture colours land where the mapping says"), ("P4", "clicks show their effect"),
                ("P5", "refusals leave the fixture as it was"), ("P6", "typed and pressed keys arrive"),
@@ -89,7 +107,7 @@ class Probe:
                 return self._skip_full(f"the fixture didn't start: {problem}")
             for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects),
                                    ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag),
-                                   ("X", self._pointer_input), ("X", self._data_add)):
+                                   ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording)):
                 await self._guarded(check, section)
         finally:
             await self._unstage()
@@ -721,6 +739,103 @@ class Probe:
 
         await self.check("X", "a file data/add writes is there for an app to read", picked_up, *DATA_PLANE)
 
+    async def _recording(self) -> None:
+        if not self.driver.declares(COMPUTER_RECORDING):
+            return
+        params = self.driver.params(COMPUTER_RECORDING)
+        fps = min(RECORD_FPS, (params.get("fps") or {}).get("max") or RECORD_FPS)
+        made: list[str] = []
+        state: dict = {}
+
+        async def start(request_id: str) -> dict:
+            started = await self.driver.invoke(COMPUTER_RECORDING, "start", {"request_id": request_id, "fps": fps})
+            if started["recording_id"] not in made:
+                made.append(started["recording_id"])
+            return started
+
+        async def recorded() -> str | None:
+            request_id = f"{PROBE_ROLE}-recording-{uuid.uuid4().hex[:8]}"
+            started = await start(request_id)
+            state["joined"] = await start(request_id) == started
+            await asyncio.sleep(RECORD_S)
+            manifest = await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": started["recording_id"]})
+            return await self._segments_match(manifest, fps)
+
+        async def joined() -> str | None:
+            return None if state.get("joined") else "a repeated request_id started a second recording"
+
+        async def limited() -> str | None:
+            most = params.get("max_concurrent")
+            if not isinstance(most, int):
+                return "the card declares no max_concurrent"
+            for _ in range(most):
+                await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}")
+            try:
+                extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}")
+            except ExtensionCallError as e:
+                return None if e.code == "limit_exceeded" else f"refused {e.code!r}, not 'limit_exceeded': {e.detail}"
+            return f"recording {most + 1} of at most {most} started: {extra['recording_id']}"
+
+        try:
+            await self.check("X", "a recording's segments match their files, with no gap between them", recorded,
+                             COMPUTER_RECORDING)
+            await self.check("X", "a repeated request_id joins the first recording", joined, COMPUTER_RECORDING)
+            await self.check("X", "a recording past max_concurrent is refused", limited, COMPUTER_RECORDING)
+        finally:
+            for recording_id in made:
+                with contextlib.suppress(Exception):
+                    await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id})
+                await self.driver.run("rm", "-rf", f"{RECORDINGS_DIR}/{recording_id}")
+            await self.driver.run("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true")
+
+    async def _segments_match(self, wire: dict, fps: int) -> str | None:
+        """What is wrong with a recording without a grant, read back from the computer, or None."""
+        try:
+            manifest = RecordingManifest.model_validate(wire)
+        except ValidationError as e:
+            return f"its manifest isn't a computer-recording/v1 manifest: {e}"
+        segments = manifest.segments
+        if manifest.ended_by != "end" or not segments:
+            return f"it ended by {manifest.ended_by!r} with {len(segments)} segments"
+        if unmeasured := [s.index for s in segments if not s.measured]:
+            return f"segments {unmeasured} lack their times, size, frame count or digest"
+        span = (_when(segments[-1].ended_at) - _when(segments[0].started_at)).total_seconds()
+        frames = sum(s.frames for s in segments)
+        if span < RECORD_S - 2 / fps or frames < fps * RECORD_S / 2:
+            return f"{frames} frames over {span:.2f} s of a {RECORD_S:g} s recording at {fps} fps"
+        for before, after in zip(segments, segments[1:]):
+            gap = (_when(after.started_at) - _when(before.ended_at)).total_seconds()
+            if gap > 2 / fps:
+                return f"{gap:.2f} s between segments {before.index} and {after.index}"
+        for segment in segments:
+            path = f"{RECORDINGS_DIR}/{segment.path}"
+            out = await self.driver.run("python3", "-c", READ_DIGEST, path)
+            if out.get("exit_code") != 0:
+                return f"segment {segment.index} couldn't be read back: {out.get('stderr', '')[-200:]}"
+            file = json.loads(out["stdout"])
+            if (file["size"], file["sha256"]) != (segment.bytes, segment.sha256):
+                return f"segment {segment.index}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \
+                       f"the manifest says {segment.bytes}, {segment.sha256[:12]}"
+            data = b""
+            while len(data) < file["size"]:
+                chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES))
+                read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b""
+                if not read:
+                    return f"segment {segment.index} stopped reading back at byte {len(data)} of {file['size']}: " \
+                           f"{chunk.get('stderr', '')[-200:]}"
+                data += read
+            mp4 = read_mp4(data)
+            span = (_when(segment.ended_at) - _when(segment.started_at)).total_seconds()
+            if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment.width, segment.height, segment.frames)
+                    or abs(mp4["duration_s"] - span) > 2 / fps):
+                return f"segment {segment.index}'s MP4 is {mp4}; the manifest says {segment.width}x" \
+                       f"{segment.height}, {segment.frames} frames over {span:.2f} s"
+        return None
+
+
+def _when(iso: str) -> datetime:
+    return datetime.fromisoformat(iso.replace("Z", "+00:00"))
+
 
 def _rgb(colour: str) -> tuple[int, int, int]:
     return int(colour[1:3], 16), int(colour[3:5], 16), int(colour[5:7], 16)

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/conformance/fixture.py

@@ -27,6 +27,7 @@ def main(state_dir: Path) -> None:
     root.title("Fixture Desk")
     width, height = root.winfo_screenwidth(), root.winfo_screenheight()
     root.geometry(f"{width}x{height}+0+0")
+    root.attributes("-fullscreen", True)
     root.configure(bg="#f0f0f0")
     state: dict = {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None,
                    "scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [],

agentenv_framework_protocol-0.1.386/src/agentenv_protocol/computer/conformance/mp4.py

@@ -0,0 +1,65 @@
+"""What the probe reads from an MP4 without decoding it: the video track's size, frame count and duration, from the
+``moov`` box's ``tkhd``, ``mdhd`` and ``stts``."""
+from __future__ import annotations
+
+import struct
+from collections.abc import Iterator
+
+
+def read_mp4(data: bytes) -> dict:
+    """``{"width", "height", "frames", "duration_s"}`` of the first video track. Raises ValueError for a file that
+    isn't an MP4 with one."""
+    top = dict(_boxes(data, 0, len(data)))
+    if b"ftyp" not in top or b"moov" not in top:
+        raise ValueError(f"not an MP4: its top-level boxes are {sorted(t.decode(errors='replace') for t in top)}")
+    for trak in _children(data, top[b"moov"], b"trak"):
+        mdia = _child(data, trak, b"mdia")
+        hdlr = _child(data, mdia, b"hdlr")
+        if data[hdlr[0] + 8:hdlr[0] + 12] != b"vide":
+            continue
+        width, height = _tkhd_size(data, _child(data, trak, b"tkhd"))
+        timescale, duration = _mdhd(data, _child(data, mdia, b"mdhd"))
+        stts = _child(data, _child(data, _child(data, mdia, b"minf"), b"stbl"), b"stts")
+        entries = struct.unpack(">I", data[stts[0] + 4:stts[0] + 8])[0]
+        frames = sum(struct.unpack(">I", data[stts[0] + 8 + 8 * i:stts[0] + 12 + 8 * i])[0] for i in range(entries))
+        return {"width": width, "height": height, "frames": frames, "duration_s": duration / timescale}
+    raise ValueError("the MP4 has no video track")
+
+
+def _boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, tuple[int, int]]]:
+    """Each box between ``start`` and ``end``: its type, and where its payload starts and ends."""
+    at = start
+    while at + 8 <= end:
+        size, kind = struct.unpack(">I4s", data[at:at + 8])
+        header = 8
+        if size == 1:
+            size, header = struct.unpack(">Q", data[at + 8:at + 16])[0], 16
+        elif size == 0:
+            size = end - at
+        if size < header or at + size > end:
+            raise ValueError(f"a {kind.decode(errors='replace')} box runs past its container")
+        yield kind, (at + header, at + size)
+        at += size
+
+
+def _children(data: bytes, box: tuple[int, int], kind: bytes) -> list[tuple[int, int]]:
+    return [span for k, span in _boxes(data, *box) if k == kind]
+
+
+def _child(data: bytes, box: tuple[int, int], kind: bytes) -> tuple[int, int]:
+    found = _children(data, box, kind)
+    if not found:
+        raise ValueError(f"no {kind.decode()} box where the MP4 needs one")
+    return found[0]
+
+
+def _tkhd_size(data: bytes, box: tuple[int, int]) -> tuple[int, int]:
+    at = box[0] + (88 if data[box[0]] == 1 else 76)
+    width, height = struct.unpack(">II", data[at:at + 8])
+    return width >> 16, height >> 16
+
+
+def _mdhd(data: bytes, box: tuple[int, int]) -> tuple[int, int]:
+    if data[box[0]] == 1:
+        return struct.unpack(">IQ", data[box[0] + 20:box[0] + 32])
+    return struct.unpack(">II", data[box[0] + 12:box[0] + 20])

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/driver.py

@@ -14,6 +14,7 @@ from .dataplane import DataPlane
 from .extensions import Extensions
 from .operations import ReadyCheck, check_class, default_for
 from .platform import Platform, detect
+from .recording import Recording
 from .runtime import ComputerRuntime
 from .settings import ComputerSettings, load_settings
 from .tools import Tools
@@ -42,6 +43,9 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen
     layouts: ClassVar[tuple[str, ...]] = ("us",)
     #: The platforms the driver supports, as the driver env declares them: ``[{"os": {...}, "arch": [...]}]``.
     supports: ClassVar[list] = []
+    #: computer-recording/v1, from ``@record`` if the driver has one, else by polling ``@capture``; ``None`` turns
+    #: it off.
+    recording: ClassVar[Recording | None] = Recording()
     tool_access = True
     #: What the base class keeps for the driver, built by ``__init__``.
     _computer: ComputerRuntime | None = None

{agentenv_framework_protocol-0.1.381 → agentenv_framework_protocol-0.1.386}/src/agentenv_protocol/computer/extensions.py

@@ -1,5 +1,5 @@
-"""The framework extensions agentenv calls, never agents: readiness, display, coordinate spaces, keyboard settings
-and commands in the session."""
+"""The framework extensions agentenv calls, never agents: readiness, display, coordinate spaces, keyboard settings,
+commands in the session and recordings."""
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Callable, Optional
@@ -11,6 +11,7 @@ from .card import (
     COMPUTER_EXEC,
     COMPUTER_KEYBOARD,
     COMPUTER_READY,
+    COMPUTER_RECORDING,
     COMPUTER_SPACES,
     KEYBOARD_RANGES,
     MAX_EXEC_TIMEOUT_S,
@@ -111,6 +112,19 @@ class Extensions:
         cap = self.settings.exec_output_bytes
         return await self._computer.command(request_id, lambda: _run(self._computer, cap, argv, cwd, env, timeout))
 
+    @extension(COMPUTER_RECORDING, op="start",
+               description="Record the display: start returns a recording whose segments upload through its grant as "
+                           "they close, and end seals it and returns its manifest; repeats answer the same.")
+    async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None,
+                                        fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True,
+                                        upload: Optional[dict] = None) -> dict:
+        return await self._computer.recordings.start(request_id, scope, window, fps, max_edge, cursor, upload)
+
+    @extension(COMPUTER_RECORDING, op="end")
+    async def _computer_recording_end(self, recording_id: str, upload: Optional[dict] = None,
+                                      reason: str = "end") -> dict:
+        return await self._computer.recordings.end(recording_id, upload, reason)
+
 
 async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict],
                timeout: float) -> dict: