agentenv-framework-protocol 0.1.439__tar.gz → 0.1.441__tar.gz

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentenv-framework-protocol
-Version: 0.1.439
+Version: 0.1.441
 Summary: Open data-plane protocol and server SDK for agent environments.
 Project-URL: Homepage, https://www.agentenvframework.com
 Project-URL: Documentation, https://www.agentenvframework.com/docs
@@ -846,8 +846,10 @@ are the profile's template. The base class adds:
   the pipe rather than a second copy, which would hand its window to the first and exit. The ready check
   `devtools_answers` asks `Browser.getVersion` over the pipe.
 - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches
-  to every page with flattened sessions and keeps a record, `since` its server started: open tabs in
-  opened order, the one whose page has focus `active`; `closed_tabs`, each with `closed_at` and
+  to every page with flattened sessions, telling each page it attached to as it opened, a popup say, to run, and keeps a
+  record, `since` its server started: open tabs in opened order, each with its URL and title as the browser has them
+  when the record is read (Chrome sends no event for a title), the one whose page has focus `active`; `closed_tabs`,
+  each as of its page's last load, with `closed_at` and
   `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser
   ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state
   and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it.

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/README.md

@@ -790,8 +790,10 @@ are the profile's template. The base class adds:
   the pipe rather than a second copy, which would hand its window to the first and exit. The ready check
   `devtools_answers` asks `Browser.getVersion` over the pipe.
 - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches
-  to every page with flattened sessions and keeps a record, `since` its server started: open tabs in
-  opened order, the one whose page has focus `active`; `closed_tabs`, each with `closed_at` and
+  to every page with flattened sessions, telling each page it attached to as it opened, a popup say, to run, and keeps a
+  record, `since` its server started: open tabs in opened order, each with its URL and title as the browser has them
+  when the record is read (Chrome sends no event for a title), the one whose page has focus `active`; `closed_tabs`,
+  each as of its page's last load, with `closed_at` and
   `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser
   ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state
   and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it.

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentenv-framework-protocol"
-version = "0.1.439"
+version = "0.1.441"
 description = "Open data-plane protocol and server SDK for agent environments."
 readme = "README.md"
 license = "Apache-2.0"

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/src/agentenv_protocol/computer/conformance/checks.py

@@ -13,6 +13,7 @@ import uuid
 from collections.abc import Awaitable, Callable
 from datetime import datetime
 from importlib import resources
+from typing import Any
 
 from mcp import ClientSession
 from mcp.types import CallToolResult
@@ -152,6 +153,10 @@ class Probe:
             raise AssertionError(f"computer_screenshot failed: {result.content[0].text}")
         return result, image(result)
 
+    async def pixel(self, x: int, y: int) -> tuple[int, int, int]:
+        """The pixel at frame pixel (x, y) of a new frame."""
+        return (await self.shot())[1].getpixel((x, y))
+
     async def act(self, name: str, args: dict) -> None:
         result = await self.tool(name, args)
         if result.isError:
@@ -161,13 +166,15 @@ class Probe:
         out = await self.driver.run("cat", f"{STAGE}/state/state.json")
         return json.loads(out["stdout"])
 
-    async def until(self, holds: Callable[[dict], bool]) -> tuple[bool, dict]:
-        """Whether the fixture's state comes to satisfy ``holds`` within SETTLE_S, and the state it last had."""
+    async def until(self, holds: Callable[[Any], bool],
+                    read: Callable[[], Awaitable[Any]] | None = None) -> tuple[bool, Any]:
+        """Whether what ``read`` gives, the fixture's state unless it's given, comes to satisfy ``holds`` within
+        SETTLE_S, and what it last gave."""
         deadline = time.monotonic() + SETTLE_S
         while True:
-            state = await self.state()
-            if holds(state) or time.monotonic() > deadline:
-                return holds(state), state
+            seen = await (read or self.state)()
+            if holds(seen) or time.monotonic() > deadline:
+                return holds(seen), seen
             await asyncio.sleep(POLL_S)
 
     async def settled(self, holds: Callable[[dict], bool], show: Callable[[dict], object]) -> str | None:
@@ -588,8 +595,8 @@ class Probe:
             problem = await self.settled(lambda s: s["cells"].get("2,3") == 1, lambda s: s["cells"])
             if problem:
                 return problem
-            _, img = await self.shot()
-            return None if img.getpixel((x, y)) == (255, 0, 0) else f"the next frame shows {img.getpixel((x, y))}"
+            shows, pixel = await self.until(lambda p: p == (255, 0, 0), lambda: self.pixel(x, y))
+            return None if shows else f"no frame within {SETTLE_S:g} s shows it: the last shows {pixel}"
 
         async def norm_space() -> str | None:
             x, y = cell(5, 5)
@@ -636,7 +643,7 @@ class Probe:
             return await self.settled(lambda s: s["saved_count"] == 1 and s["subscribed"],
                                       lambda s: (s["saved_count"], s["subscribed"]))
 
-        await self.check("P4", "a click in frame space hits its cell, and the next frame shows it", frame_space)
+        await self.check("P4", "a click in frame space hits its cell, and a frame shows it", frame_space)
         if self.driver.declares(COMPUTER_SPACES):
             await self.check("P4", "a click in norm space hits its cell", norm_space, COMPUTER_SPACES)
         if "right" in buttons:

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/src/agentenv_protocol/website_browser.py

@@ -260,13 +260,26 @@ class BrowserControl:
         return visible
 
     async def record(self) -> dict:
-        """The record ``since`` it started: open ``tabs``, each with whether it's ``active``, ``closed_tabs``,
-        ``downloads`` and ``dialogs``."""
+        """The record ``since`` it started: open ``tabs``, each with its URL and title now and whether it's ``active``,
+        ``closed_tabs``, ``downloads`` and ``dialogs``."""
+        await self._refresh()
         active = await self.active_tab()
         return {"since": self.since, "tabs": [{**tab, "active": tab["id"] == active} for tab in self.tabs.values()],
                 "closed_tabs": list(self.closed_tabs), "downloads": list(self.downloads.values()),
                 "dialogs": list(self.dialog_log)}
 
+    async def _refresh(self, target: Optional[str] = None) -> None:
+        """The URL and title of the tab ``target``, or of every open tab, as the browser has them now: it sends no
+        event when a page's title changes."""
+        try:
+            infos = (await self.devtools.call("Target.getTargets", {"filter": PAGES},
+                                              timeout_s=ACTIVE_CHECK_TIMEOUT_S))["targetInfos"]
+        except (ConnectionError, DevToolsError, asyncio.TimeoutError, KeyError):
+            return
+        for info in infos:
+            if target in (None, info["targetId"]) and (tab := self.tabs.get(info["targetId"])) is not None:
+                tab.update(url=info.get("url", tab["url"]), title=info.get("title", tab["title"]))
+
     def _event(self, message: dict) -> None:
         handler = getattr(self, "_on_" + message.get("method", "").replace(".", "_"), None)
         if handler is not None:
@@ -295,10 +308,16 @@ class BrowserControl:
         if params["targetInfo"].get("type") == "page":
             self._sessions[params["sessionId"]] = params["targetInfo"]["targetId"]
             self._spawn(self.devtools.call("Page.enable", session_id=params["sessionId"]))
+            # A page attached as it opens, a popup say, waits for this before it navigates.
+            self._spawn(self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=params["sessionId"]))
 
     def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None:
         self._sessions.pop(params["sessionId"], None)
 
+    def _on_Page_loadEventFired(self, params: dict, session: Optional[str]) -> None:
+        if (target := self._sessions.get(session)) is not None:
+            self._spawn(self._refresh(target))
+
     def _on_Page_javascriptDialogOpening(self, params: dict, session: Optional[str]) -> None:
         dialog = {"tab": self._sessions.get(session), "type": params.get("type"), "message": params.get("message", ""),
                   "url": params.get("url", ""), "opened_at": _now(), "answer": None, "by": None}

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/tests/test_computer_conformance.py

@@ -85,12 +85,43 @@ async def test_a_driver_given_a_control_token_passes_with_it_and_refuses_calls_w
     assert not unauthorized.passed and "401 unauthorized" in _failed(unauthorized)[0]
 
 
+class StaleFrame(FixtureDriver):
+    """Its first ``stale`` captures after a click are from before the click, as when an app records a click before it
+    repaints."""
+
+    def __init__(self, stale: int = 1, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.stale, self.left, self.before_click = stale, 0, None
+
+    @click
+    async def click_at(self, x, y, button, count, modifiers):
+        self.before_click, self.left = self.screen.image.copy(), self.stale
+        return await super().click_at(x, y, button, count, modifiers)
+
+    @capture
+    async def grab(self):
+        if not self.left:
+            return self.screen.image.copy()
+        self.left -= 1
+        return self.before_click.copy()
+
+
 @pytest.mark.asyncio
-async def test_a_driver_whose_default_space_is_norm_passes_at_full():
-    report = await _probe(FixtureDriver(settings=ComputerSettings(default_space="norm")))
+@pytest.mark.parametrize("driver", [lambda: FixtureDriver(settings=ComputerSettings(default_space="norm")), StaleFrame],
+                         ids=["default-space-norm", "stale-first-frame"])
+async def test_a_driver_whose_default_space_is_norm_or_whose_first_frame_after_a_click_is_stale_passes_at_full(driver):
+    report = await _probe(driver())
     assert report.passed, _failed(report)
 
 
+@pytest.mark.asyncio
+async def test_frames_that_never_show_a_click_the_fixture_saw_fail_p4_with_the_last_pixel(monkeypatch):
+    monkeypatch.setattr(checks, "SETTLE_S", 0.5)
+    report = await _probe(StaleFrame(stale=10**6))
+    failed = next(c for c in report.checks if c.name == "a click in frame space hits its cell, and a frame shows it")
+    assert (failed.status, failed.detail) == ("failed", "no frame within 0.5 s shows it: the last shows (255, 255, 255)")
+
+
 @pytest.mark.asyncio
 async def test_a_tool_already_hidden_from_the_probes_role_stays_hidden():
     with served(FixtureDriver()) as url:
@@ -195,7 +226,7 @@ class ExitZero(FixtureDriver):
 @pytest.mark.asyncio
 @pytest.mark.parametrize("driver, broken", [
     (NeverReady, "computer-ready reports ready"),
-    (ClicksOff, "a click in frame space hits its cell, and the next frame shows it"),
+    (ClicksOff, "a click in frame space hits its cell, and a frame shows it"),
     (ShiftedCapture, "markers at odd and even pixels read back in frame pixels"),
     (AsciiOnly, "Unicode and mixed-case text arrives whole"),
     (SwappedScroll, "scrolls up, down, left and right each arrive that many times"),

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/tests/test_computer_conformance_checks.py

@@ -229,7 +229,7 @@ async def test_effects_without_spaces_right_or_double_clicks_leave_those_out_and
     assert "a click in norm space hits its cell" not in got and "a right click arrives as one" not in got
     assert "a double click arrives as one" not in got and "norm points land on the same pixels" not in got
     assert got["16 clicks sent at once all land"] == ("failed", "16 of 16 were refused")
-    assert got["a click in frame space hits its cell, and the next frame shows it"][1].startswith("AssertionError")
+    assert got["a click in frame space hits its cell, and a frame shows it"][1].startswith("AssertionError")
 
 
 @pytest.mark.asyncio

{agentenv_framework_protocol-0.1.439 → agentenv_framework_protocol-0.1.441}/tests/test_website_browser.py

@@ -3,7 +3,8 @@ for each; a subclass refused for carrying one of them; the start tabs, the time
 its launch run with no desktop when alone; a CDP connection over a new debugging pipe at each launch, which its ready
 check asks, and a browser launched again when its server restarts; its shared memory in files only on a small
 /dev/shm; an empty cache and the deploy's prompt answers in its profile at each launch; its policies, card params and
-skill; and an app that isn't a browser launched and declared as before."""
+skill; its record's titles read from the browser, which sends no event for a page's; and an app that isn't a browser
+launched and declared as before."""
 from __future__ import annotations
 
 import asyncio
@@ -492,6 +493,40 @@ async def test_its_control_records_tabs_in_opened_order_and_who_closed_each_and_
     assert devtools.calls[2][1] == {"behavior": "default", "eventsEnabled": True}
 
 
+@pytest.mark.asyncio
+async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to_as_it_opened_navigates(tmp_path):
+    devtools = StubDevTools()
+    control = BrowserControl(devtools, "agent")
+    await control.attach()
+    devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")})
+    await asyncio.sleep(0)
+
+    assert ("Page.enable", {}, "s-popup") in devtools.calls
+    assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls
+
+
+@pytest.mark.asyncio
+async def test_its_record_reads_each_tabs_title_when_read_and_when_its_page_loads_since_no_event_says(tmp_path):
+    now = {"a": "Inbox", "b": "Draft"}
+    devtools = StubDevTools({"Target.getTargets": lambda params, session: {"targetInfos": [
+        _page(target, f"https://mail.test/{target}", title)["targetInfo"] for target, title in now.items()]}})
+    control = BrowserControl(devtools, "agent")
+    await control.attach()
+    for target in ("a", "b"):
+        devtools.send("Target.targetCreated", _page(target, f"https://mail.test/{target}", f"mail.test/{target}"))
+        devtools.send("Target.attachedToTarget", {"sessionId": f"s-{target}", **_page(target)})
+    devtools.send("Page.loadEventFired", {"timestamp": 1}, session="s-b")
+    await asyncio.sleep(0)
+    del now["b"]
+    devtools.send("Target.targetDestroyed", {"targetId": "b"})
+    now["a"] = "Inbox (1)"
+    record = await control.record()
+
+    assert [(tab["id"], tab["title"]) for tab in record["tabs"]] == [("a", "Inbox (1)")]
+    assert [(tab["id"], tab["title"]) for tab in record["closed_tabs"]] == [("b", "Draft")]
+    assert ("Target.getTargets", {"filter": [{"type": "page"}]}, None) in devtools.calls
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("dialogs, kind, sent, answer, by", [
     ("accept", "prompt", {"accept": True, "promptText": "yes"}, "accepted", "agentenv"),