agentenv-framework-protocol 0.1.443__tar.gz → 0.1.445__tar.gz

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/.gitignore

@@ -50,6 +50,7 @@ __pycache__/
 build/
 develop-eggs/
 dist/
+node_modules/
 downloads/
 eggs/
 .eggs/

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentenv-framework-protocol
-Version: 0.1.443
+Version: 0.1.445
 Summary: Open data-plane protocol and server SDK for agent environments.
 Project-URL: Homepage, https://www.agentenvframework.com
 Project-URL: Documentation, https://www.agentenvframework.com/docs
@@ -529,6 +529,7 @@ pixels), at the display's size times its scale. Input operations may return an `
 | `@set_display` | no, with `display_modes` | `computer-display/v1` `set` (`get` is always served) |
 | `@pointer_move`, `@pointer_down`, `@pointer_up`, `@pointer_position` | all or none | `computer-pointer/v1` and its five tools; click and scroll at the pointer |
 | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools |
+| `@navigate`, `@history`, `@tabs`, `@dialog`, `@upload`, `@page_state` | all or none, for a screen that is a website browser's page | `computer-page/v1` and its five tools; the `page` of every frame |
 | `@run_in_session` | no; a Linux default | `computer-exec/v1` |
 | `@write_files` | no; a Linux default | `data/add` |
 | `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` |
@@ -553,15 +554,24 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
   group tools. Every result is structured, with a one-line summary as its text block; input results
   carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame
   after acting. Click and scroll leave out `x` and `y` to act at the pointer on a computer with
-  `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`.
+  `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. With the page
+  group, `computer_navigate` opens a URL in the active tab, `computer_history` goes `back`, `forward`
+  or `reload`s, `computer_tabs` lists, opens, switches and closes tabs by the browser's own ids and
+  answers with the tabs after it, `computer_dialog` answers the page's open dialog and
+  `computer_upload` gives its open file chooser files in `~/Uploads`.
 - **Frames and spaces.** Points are pixels of a named frame (your latest by default), or a
   0..`norm_scale` grid over it with `space: "norm"`. A point outside the frame or the range is
   refused (`out_of_bounds`), never clamped. The last 64 frames are kept, and a frame goes stale
-  (`stale_frame`) when the display geometry it was captured at changes.
+  (`stale_frame`) when the display geometry it was captured at changes. With the page group, every
+  frame carries `page`: the active tab's `tab`, `url` and `title`, and the `dialog` (`type`, `message`,
+  `default_text`) or `file_chooser` (`multiple`) open on it, which its summary names too.
 - **The input queue.** Input calls run one at a time in arrival order; one that waits past
-  `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue.
+  `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. While a dialog is open
+  on the page, input it blocks (all but `computer_tabs`, `computer_dialog` and `computer_upload`) is
+  refused `dialog_open`, with `applied: "no"`.
 - **The computer skill.** `computer/v1` declares the seven tools, its params the driver's
-  features. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base
+  features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website
+  browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base
   class): a SKILL.md written from the computer as it booted (its platform, session user and home,
   the live screen size, what its input takes, its OS's shortcuts) and ending with the driver
   class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of
@@ -569,7 +579,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
   computer skill leads with it, the computer's part following.
 - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a
   capture, the driver's own `ready_checks()`, and one per app on the computer, asked of the app's
-  own server (see Computer apps); tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool
+  own server (see Computer apps); tools refuse `device_unavailable` until they pass once; a check
+  that can't pass again is `final`, and while one fails the status carries `failed`, saying why,
+  which `boot.ready()` raises at once rather than waiting out its timeout), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool
   descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?,
   request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and,
   with `recording`, `computer-recording/v1` (below).
@@ -798,7 +810,7 @@ its card lists each one's card as a child environment, rooted at `/svc/<app>/`,
 requests to the app's server.
 
 `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app`
-env stores, and `python -m agentenv_protocol.computer._declare <folder> <file> <class>` prints it as
+env stores (`browser: true` for a website browser), and `python -m agentenv_protocol.computer._declare <folder> <file> <class>` prints it as
 JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed.
 
 ### Website browsers
@@ -825,8 +837,8 @@ are the profile's template. The base class adds:
   `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off.
   Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`,
   `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go
-  through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`,
-  except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy.
+  through. Alone, on a computer with no display (`XDG_SESSION_TYPE=none`, a browser computer's), it
+  also takes `--window-size` and `--headless=new`, and refuses to start with no proxy.
   `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, or
   any other way to open a debugging port.
 - **Before each launch** it empties its cache, so no launch inherits another's, and writes the deploy's
@@ -852,14 +864,28 @@ are the profile's template. The base class adds:
   each as of its page's last load, with `closed_at` and
   `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser
   ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state
-  and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it.
+  and, completed, its size and sha256; and `dialogs`, each with its type, message, default text, answer
+  and who gave it.
   `hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record.
 - **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and
   `record` (GET), framework-only, which returns that record with `launches`, the app server's log of each
   launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made
   and each exit it didn't, with its exit `code` or the `signal` that ended it, since the computer booted.
   Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra
-  failure, its ready check false.
+  failure, its ready check false. `page/viewport`, `page/capture` and `page/input`, framework-only, are
+  its active tab's page as a page driver sees it: its size, which headless is its window's; a PNG of it,
+  or `blank` for a tab that paints nothing within 3 s (1 s with no URL yet), its last capture while
+  a dialog keeps it from painting; and CDP input calls (`Input.dispatchMouseEvent`,
+  `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. Also framework-only, for page
+  drivers and agentenv's own steps alike:
+  - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab;
+  - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab
+    ids. The active tab is the one a dialog is open on, else the one whose page has focus, else the one
+    agentenv last brought to the front;
+  - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab;
+  - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so
+    alone, it intercepts;
+  - `page/state`, the active tab's id, URL and title and the dialog or file chooser open on it.
 - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in
   `~/Downloads`, folders it makes.
 

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/README.md

@@ -473,6 +473,7 @@ pixels), at the display's size times its scale. Input operations may return an `
 | `@set_display` | no, with `display_modes` | `computer-display/v1` `set` (`get` is always served) |
 | `@pointer_move`, `@pointer_down`, `@pointer_up`, `@pointer_position` | all or none | `computer-pointer/v1` and its five tools; click and scroll at the pointer |
 | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools |
+| `@navigate`, `@history`, `@tabs`, `@dialog`, `@upload`, `@page_state` | all or none, for a screen that is a website browser's page | `computer-page/v1` and its five tools; the `page` of every frame |
 | `@run_in_session` | no; a Linux default | `computer-exec/v1` |
 | `@write_files` | no; a Linux default | `data/add` |
 | `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` |
@@ -497,15 +498,24 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
   group tools. Every result is structured, with a one-line summary as its text block; input results
   carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame
   after acting. Click and scroll leave out `x` and `y` to act at the pointer on a computer with
-  `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`.
+  `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. With the page
+  group, `computer_navigate` opens a URL in the active tab, `computer_history` goes `back`, `forward`
+  or `reload`s, `computer_tabs` lists, opens, switches and closes tabs by the browser's own ids and
+  answers with the tabs after it, `computer_dialog` answers the page's open dialog and
+  `computer_upload` gives its open file chooser files in `~/Uploads`.
 - **Frames and spaces.** Points are pixels of a named frame (your latest by default), or a
   0..`norm_scale` grid over it with `space: "norm"`. A point outside the frame or the range is
   refused (`out_of_bounds`), never clamped. The last 64 frames are kept, and a frame goes stale
-  (`stale_frame`) when the display geometry it was captured at changes.
+  (`stale_frame`) when the display geometry it was captured at changes. With the page group, every
+  frame carries `page`: the active tab's `tab`, `url` and `title`, and the `dialog` (`type`, `message`,
+  `default_text`) or `file_chooser` (`multiple`) open on it, which its summary names too.
 - **The input queue.** Input calls run one at a time in arrival order; one that waits past
-  `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue.
+  `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. While a dialog is open
+  on the page, input it blocks (all but `computer_tabs`, `computer_dialog` and `computer_upload`) is
+  refused `dialog_open`, with `applied: "no"`.
 - **The computer skill.** `computer/v1` declares the seven tools, its params the driver's
-  features. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base
+  features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website
+  browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base
   class): a SKILL.md written from the computer as it booted (its platform, session user and home,
   the live screen size, what its input takes, its OS's shortcuts) and ending with the driver
   class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of
@@ -513,7 +523,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
   computer skill leads with it, the computer's part following.
 - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a
   capture, the driver's own `ready_checks()`, and one per app on the computer, asked of the app's
-  own server (see Computer apps); tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool
+  own server (see Computer apps); tools refuse `device_unavailable` until they pass once; a check
+  that can't pass again is `final`, and while one fails the status carries `failed`, saying why,
+  which `boot.ready()` raises at once rather than waiting out its timeout), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool
   descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?,
   request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and,
   with `recording`, `computer-recording/v1` (below).
@@ -742,7 +754,7 @@ its card lists each one's card as a child environment, rooted at `/svc/<app>/`,
 requests to the app's server.
 
 `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app`
-env stores, and `python -m agentenv_protocol.computer._declare <folder> <file> <class>` prints it as
+env stores (`browser: true` for a website browser), and `python -m agentenv_protocol.computer._declare <folder> <file> <class>` prints it as
 JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed.
 
 ### Website browsers
@@ -769,8 +781,8 @@ are the profile's template. The base class adds:
   `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off.
   Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`,
   `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go
-  through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`,
-  except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy.
+  through. Alone, on a computer with no display (`XDG_SESSION_TYPE=none`, a browser computer's), it
+  also takes `--window-size` and `--headless=new`, and refuses to start with no proxy.
   `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, or
   any other way to open a debugging port.
 - **Before each launch** it empties its cache, so no launch inherits another's, and writes the deploy's
@@ -796,14 +808,28 @@ are the profile's template. The base class adds:
   each as of its page's last load, with `closed_at` and
   `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser
   ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state
-  and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it.
+  and, completed, its size and sha256; and `dialogs`, each with its type, message, default text, answer
+  and who gave it.
   `hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record.
 - **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and
   `record` (GET), framework-only, which returns that record with `launches`, the app server's log of each
   launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made
   and each exit it didn't, with its exit `code` or the `signal` that ended it, since the computer booted.
   Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra
-  failure, its ready check false.
+  failure, its ready check false. `page/viewport`, `page/capture` and `page/input`, framework-only, are
+  its active tab's page as a page driver sees it: its size, which headless is its window's; a PNG of it,
+  or `blank` for a tab that paints nothing within 3 s (1 s with no URL yet), its last capture while
+  a dialog keeps it from painting; and CDP input calls (`Input.dispatchMouseEvent`,
+  `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. Also framework-only, for page
+  drivers and agentenv's own steps alike:
+  - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab;
+  - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab
+    ids. The active tab is the one a dialog is open on, else the one whose page has focus, else the one
+    agentenv last brought to the front;
+  - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab;
+  - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so
+    alone, it intercepts;
+  - `page/state`, the active tab's id, URL and title and the dialog or file chooser open on it.
 - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in
   `~/Downloads`, folders it makes.
 

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentenv-framework-protocol"
-version = "0.1.443"
+version = "0.1.445"
 description = "Open data-plane protocol and server SDK for agent environments."
 readme = "README.md"
 license = "Apache-2.0"

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/__init__.py

@@ -32,6 +32,7 @@ from .errors import (
     ComputerError,
     DeliveryFailed,
     DeviceUnavailable,
+    DialogOpen,
     InputBusy,
     Internal,
     InvalidArgument,
@@ -47,14 +48,22 @@ from .operations import (
     Display,
     Effect,
     ExecResult,
+    FileChooser,
     Image,
+    PageDialog,
+    PageState,
     ReadyCheck,
+    Tab,
     capture,
     click,
+    dialog,
     display,
     drag,
+    history,
     key_down,
     key_up,
+    navigate,
+    page_state,
     pointer_down,
     pointer_move,
     pointer_position,
@@ -64,7 +73,9 @@ from .operations import (
     run_in_session,
     scroll,
     set_display,
+    tabs,
     type_text,
+    upload,
     write_files,
 )
 from .dataplane import part_bytes
@@ -79,6 +90,7 @@ if TYPE_CHECKING:
         COMPUTER_DISPLAY,
         COMPUTER_EXEC,
         COMPUTER_KEYBOARD,
+        COMPUTER_PAGE,
         COMPUTER_POINTER,
         COMPUTER_READY,
         COMPUTER_RECORDING,
@@ -91,7 +103,7 @@ if TYPE_CHECKING:
 _ON_USE = {
     "driver": ("ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration"),
     "card": ("COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER",
-             "COMPUTER_KEYBOARD", "COMPUTER_EXEC", "COMPUTER_RECORDING"),
+             "COMPUTER_KEYBOARD", "COMPUTER_PAGE", "COMPUTER_EXEC", "COMPUTER_RECORDING"),
     "recording": ("Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread"),
 }
 
@@ -110,14 +122,17 @@ __all__ = [
     "AppDeclaration", "AppOs",
     "APP_SETTINGS_ENV", "part_bytes",
     "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV",
+    "Tab", "PageState", "PageDialog", "FileChooser",
     "PRINTABLE_KEYS",
     "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press",
     "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up",
+    "navigate", "history", "tabs", "dialog", "upload", "page_state",
     "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment",
     "frames_in_thread", "RecordingManifest", "ManifestSegment",
     "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported",
-    "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed",
+    "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "DialogOpen", "LimitExceeded", "Timeout",
+    "DeliveryFailed",
     "Internal",
     "COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD",
-    "COMPUTER_EXEC", "COMPUTER_RECORDING",
+    "COMPUTER_PAGE", "COMPUTER_EXEC", "COMPUTER_RECORDING",
 ]

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/app.py

@@ -236,7 +236,7 @@ class AppDeclaration(BaseModel):
     """What a ComputerAppEnvironment class declares, as a ``computer_app`` env stores it: the platforms it ``supports``,
     as a driver names them; ``os``, what it declares for each OS type; ``settings`` and ``build_settings``, the JSON
     schemas of its ``Settings`` and of its ``BuildSettings`` when it declares any; its ``skill``; and the ``data``
-    methods, ``install_steps`` and ``ready_checks`` it serves, by name."""
+    methods, ``install_steps`` and ``ready_checks`` it serves, by name; and ``browser``, true for a website browser."""
 
     model_config = ConfigDict(extra="forbid")
 
@@ -249,6 +249,7 @@ class AppDeclaration(BaseModel):
     data: list[str]
     install_steps: list[InstallStep]
     ready_checks: list[AppReadyCheck]
+    browser: Optional[bool] = None
 
     @model_validator(mode="before")
     @classmethod

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/apps.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import asyncio
 import copy
+import json
 from pathlib import Path
 from typing import Optional
 
@@ -16,7 +17,7 @@ import httpx
 from .. import client
 from ..types import RPC_PATH, WELL_KNOWN_PATH, EnvironmentCard
 from .app import COMPUTER_APP, FAILURE, SOCKET
-from .app_runtime import CHECK_TIMEOUT_S, AppReadiness
+from .app_runtime import CHECK_TIMEOUT_S, LAUNCHES, AppReadiness
 from .operations import ReadyCheck
 
 #: Where a composed computer installs its apps, a folder each.
@@ -54,7 +55,7 @@ async def check(folder: Path) -> ReadyCheck:
 async def children(folders: list[Path]) -> list[EnvironmentCard]:
     """Each app in ``folders`` as a child environment of its computer: its server's card, rooted at ``/svc/<name>/``,
     or, while its server isn't answering, its name and route alone."""
-    cards = await asyncio.gather(*(_card(folder) for folder in folders))
+    cards = await asyncio.gather(*(served_card(folder) for folder in folders))
     return [EnvironmentCard.model_validate(child_card(folder.name, card)) if card is not None
             else EnvironmentCard(name=folder.name, url=f"/svc/{folder.name}{RPC_PATH}")
             for folder, card in zip(folders, cards)]
@@ -85,7 +86,7 @@ def child_card(name: str, card: dict) -> dict:
 
 async def owned_paths(folders: list[Path]) -> list[str]:
     """The paths under the session user's home that the apps in ``folders`` keep their data in, as their cards say."""
-    cards = await asyncio.gather(*(_card(folder) for folder in folders))
+    cards = await asyncio.gather(*(served_card(folder) for folder in folders))
     return [path for card in cards if card is not None
             for path in client.extension_params(card, COMPUTER_APP).get("paths") or []]
 
@@ -104,7 +105,7 @@ def endpoint(card: dict, method: str) -> str:
     return client.find_extension_method(card, COMPUTER_APP, method)["endpoint"]
 
 
-async def _card(folder: Path) -> Optional[dict]:
+async def served_card(folder: Path) -> Optional[dict]:
     """The card the app in ``folder`` serves, or None while its server isn't answering."""
     try:
         async with _client(folder / SOCKET) as http:
@@ -113,6 +114,33 @@ async def _card(folder: Path) -> Optional[dict]:
         return None
 
 
+def ended_on_its_own(folder: Path) -> Optional[str]:
+    """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says it
+    ended on its own, an infrastructure failure; None while it hasn't."""
+    for entry in launch_log(folder):
+        if entry.get("event") == "exit":
+            how = (f"by signal {entry['signal']}" if "signal" in entry else
+                   f"with code {entry['code']}" if "code" in entry else "in a way that wasn't recorded")
+            return (f"it ended on its own at {entry.get('at')}, {how}, and nothing launches it again: an "
+                    "infrastructure failure")
+    return None
+
+
+def launch_log(folder: Path) -> list[dict]:
+    """Each entry of the launch log of the app in ``folder`` that's whole, in order."""
+    try:
+        lines = (folder / LAUNCHES).read_text(encoding="utf-8").splitlines()
+    except FileNotFoundError:
+        return []
+    entries = []
+    for line in lines:
+        try:
+            entries.append(json.loads(line))
+        except ValueError:
+            continue
+    return entries
+
+
 def _unanswered(folder: Path) -> str:
     try:
         return f"its server failed to start: {(folder / FAILURE).read_text()}"

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/card.py

@@ -18,6 +18,7 @@ COMPUTER_DISPLAY = "urn:agentenv:computer-display/v1"
 COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1"
 COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1"
 COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1"
+COMPUTER_PAGE = "urn:agentenv:computer-page/v1"
 COMPUTER_EXEC = "urn:agentenv:computer-exec/v1"
 COMPUTER_RECORDING = "urn:agentenv:computer-recording/v1"
 
@@ -29,6 +30,7 @@ GROUP_TOOLS = {
     "pointer": ("computer_pointer_move", "computer_pointer_down", "computer_pointer_up", "computer_pointer_position",
                 "computer_pointer_list"),
     "keyboard": ("computer_keyboard_down", "computer_keyboard_up"),
+    "page": ("computer_navigate", "computer_history", "computer_tabs", "computer_dialog", "computer_upload"),
 }
 #: The tools whose description names the default coordinate space.
 POINT_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_pointer_move",
@@ -89,7 +91,7 @@ def computer_features(driver: ComputerDriverEnvironment) -> dict:
         "max_wait_ms": driver.max_wait_ms, "max_drag_points": driver.max_drag_points,
         "max_duration_ms": driver.max_duration_ms, "max_key_repeat": driver.max_key_repeat,
         "max_scroll_amount": driver.max_scroll_amount, "delivery": "foreground", "platform": driver.platform.as_dict(),
-        "supports": list(driver.supports), "default_data_paths": list(DEFAULT_DATA_PATHS),
+        "supports": list(driver.supports), "default_data_paths": list(DEFAULT_DATA_PATHS), "screen": driver.screen_kind,
     }
 
 
@@ -101,6 +103,12 @@ def declared_extensions(driver: ComputerDriverEnvironment) -> list[EnvironmentEx
         declared.append(EnvironmentExtension(
             uri=COMPUTER_POINTER, description="The computer_pointer_* tools, and click and scroll at the pointer.",
             params={"buttons": list(driver.buttons), "kinds": ["foreground"]}))
+    if driver.has("page"):
+        declared.append(EnvironmentExtension(
+            uri=COMPUTER_PAGE, description="The page tools, for a screen that is a website browser's active tab: "
+                                           "computer_navigate, computer_history, computer_tabs, computer_dialog and "
+                                           "computer_upload; each frame names the page it shows.",
+            params={"uploads": "~/Uploads"}))
     intake = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")])
     if driver.has("write_files"):
         intake.add = [IntakeFormat(part="file", format="file-tree", load="additive")]

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/checks.py

@@ -37,6 +37,8 @@ def check_features(cls: type) -> None:
                 problems.append("keys are W3C key values in their W3C case")
         except AgentEnvError as e:
             problems.append(f"keys: {e.message}")
+    if cls.screen_kind not in ("desktop", "page"):
+        problems.append(f"screen_kind is desktop or page, got {cls.screen_kind!r}")
     if ("set_display" in declared(cls)) != bool(cls.display_modes):
         problems.append("display_modes and @set_display come together")
     problems += check_recording(cls.recording)

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/driver.py

@@ -4,7 +4,7 @@ computer's apps as child environments, each reached at ``/svc/<name>/`` through
 from __future__ import annotations
 
 import asyncio
-from typing import Any, Callable, ClassVar
+from typing import Any, Callable, ClassVar, Literal
 
 import httpx
 from pydantic import BaseModel, ConfigDict
@@ -79,6 +79,9 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen
     recording: ClassVar[Recording | None] = Recording()
     #: What an agent should know about this driver's computer beyond what its card says, as markdown: the end of its skill.
     notes: ClassVar[str] = ""
+    #: What the computer's screen shows, its card's ``screen``: a ``desktop``, or a website browser's ``page``, its
+    #: active tab.
+    screen_kind: ClassVar[Literal["desktop", "page"]] = "desktop"
     tool_access = True
     #: What the base class keeps for the driver, built by ``__init__``.
     _computer: ComputerRuntime | None = None

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/errors.py

@@ -55,6 +55,10 @@ class DeviceUnavailable(ComputerError):
     CODE, STATUS, APPLIED, RETRYABLE = "device_unavailable", 503, "no", True
 
 
+class DialogOpen(ComputerError):
+    CODE, STATUS, APPLIED = "dialog_open", 409, "no"
+
+
 class LimitExceeded(ComputerError):
     CODE, STATUS, APPLIED, RETRYABLE = "limit_exceeded", 429, "no", True
 

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/extensions.py

@@ -45,8 +45,12 @@ class Extensions:
                description="Whether the computer is usable now, by real checks; never sends input.")
     async def _computer_ready_status(self) -> dict:
         checks = await self._computer.ready_checks()
-        return {"ready": all(c.ok for c in checks),
-                "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]}
+        status = {"ready": all(c.ok for c in checks),
+                  "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail, **({"final": True} if c.final else {})}
+                             for c in checks]}
+        if failed := [c for c in checks if c.final and not c.ok]:
+            status["failed"] = "; ".join(f"{c.name}: {c.detail}" for c in failed)
+        return status
 
     @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.")
     async def _computer_display_get(self) -> dict:

{agentenv_framework_protocol-0.1.443 → agentenv_framework_protocol-0.1.445}/src/agentenv_protocol/computer/frames.py

@@ -11,8 +11,8 @@ from datetime import datetime, timezone
 from PIL import Image as PILImage
 
 from .errors import Internal, StaleFrame
-from .operations import Display, Image
-from .results import DisplayData, FrameData
+from .operations import Display, Image, PageState
+from .results import DialogData, DisplayData, FileChooserData, FrameData, PageData
 
 KEEP = 64
 #: The JPEG quality a screenshot answers with when it names none.
@@ -34,6 +34,8 @@ class Frame:
     parent_id: str | None
     captured_at: datetime
     inputs: tuple = field(repr=False)
+    #: The page it shows, on a computer whose screen is a website browser's active tab.
+    page: PageState | None = None
 
     @property
     def width(self) -> int:
@@ -71,8 +73,21 @@ class Frame:
             display=DisplayData(width=self.display.width, height=self.display.height, scale=self.display.scale),
             parent_frame_id=self.parent_id,
             captured_at=self.captured_at.isoformat().replace("+00:00", "Z"),
+            page=_page_data(self.page),
         )
 
+    def note(self) -> str:
+        """What a summary adds for the page it shows: its title and URL, and an open dialog or file chooser."""
+        if self.page is None:
+            return ""
+        page = self.page
+        note = f' on "{page.title}" ({page.url})'
+        if page.dialog is not None:
+            note += f'; a {page.dialog.type} dialog is open: "{page.dialog.message}"'
+        if page.file_chooser is not None:
+            note += f"; a file chooser is open, for {'files' if page.file_chooser.multiple else 'one file'}"
+        return note
+
     def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str]:
         """The frame as base64 and its MIME type."""
         if fmt != "jpeg":
@@ -92,7 +107,7 @@ class FrameStore:
         self._latest: dict[str, str] = {}
 
     def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identity: str,
-            parent: Frame | None = None, origin: tuple[int, int] = (0, 0)) -> Frame:
+            parent: Frame | None = None, origin: tuple[int, int] = (0, 0), page: PageState | None = None) -> Frame:
         frame = Frame(
             id=f"f{self._next}",
             png=_png(image),
@@ -103,6 +118,7 @@ class FrameStore:
             parent_id=parent.id if parent is not None else None,
             captured_at=parent.captured_at if parent is not None else datetime.now(timezone.utc),
             inputs=parent.inputs if parent is not None else inputs,
+            page=parent.page if parent is not None else page,
         )
         self._next += 1
         self._frames[frame.id] = frame
@@ -153,6 +169,16 @@ def to_pil(capture: object, display: Display) -> PILImage.Image:
     return image
 
 
+def _page_data(page: PageState | None) -> PageData | None:
+    if page is None:
+        return None
+    dialog, chooser = page.dialog, page.file_chooser
+    return PageData(tab=page.tab, url=page.url, title=page.title,
+                    dialog=DialogData(type=dialog.type, message=dialog.message, default_text=dialog.default_text)
+                    if dialog is not None else None,
+                    file_chooser=FileChooserData(multiple=chooser.multiple) if chooser is not None else None)
+
+
 def _png(image: PILImage.Image) -> bytes:
     buffer = io.BytesIO()
     image.save(buffer, format="PNG")