fleet-python 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

fleet/{manager → instance}/client.py

@@ -1,8 +1,9 @@
 """Fleet SDK Base Environment Classes."""
 
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 import asyncio
 import httpx
+import inspect
 import time
 import logging
 from urllib.parse import urlparse
@@ -11,6 +12,8 @@ from ..resources.sqlite import AsyncSQLiteResource
 from ..resources.browser import AsyncBrowserResource
 from ..resources.base import Resource
 
+from ..verifiers import DatabaseSnapshot
+
 from ..exceptions import FleetEnvironmentError, FleetAPIError
 
 from .base import SyncWrapper, AsyncWrapper
@@ -20,6 +23,8 @@ from .models import (
     Resource as ResourceModel,
     ResourceType,
     HealthResponse,
+    ExecuteFunctionRequest,
+    ExecuteFunctionResponse,
 )
 
 
@@ -31,6 +36,11 @@ RESOURCE_TYPES = {
     ResourceType.cdp: AsyncBrowserResource,
 }
 
+ValidatorType = Callable[
+    [DatabaseSnapshot, DatabaseSnapshot, Optional[str]],
+    int,
+]
+
 
 class InstanceClient:
     def __init__(
@@ -57,7 +67,8 @@ class AsyncInstanceClient:
     ):
         self.base_url = url
         self.client = AsyncWrapper(
-            url=self.base_url, httpx_client=httpx_client or httpx.AsyncClient()
+            url=self.base_url,
+            httpx_client=httpx_client or httpx.AsyncClient(timeout=60.0),
         )
         self._resources: Optional[List[ResourceModel]] = None
         self._resources_state: Dict[str, Dict[str, Resource]] = {
@@ -106,15 +117,41 @@ class AsyncInstanceClient:
             for resource in resources_by_name.values()
         ]
 
+    async def verify(self, validator: ValidatorType) -> ExecuteFunctionResponse:
+        function_code = inspect.getsource(validator)
+        function_name = validator.__name__
+        return await self.verify_raw(function_code, function_name)
+
+    async def verify_raw(
+        self, function_code: str, function_name: str
+    ) -> ExecuteFunctionResponse:
+        response = await self.client.request(
+            "POST",
+            "/execute_verifier_function",
+            json=ExecuteFunctionRequest(
+                function_code=function_code,
+                function_name=function_name,
+            ).model_dump(),
+        )
+        return ExecuteFunctionResponse(**response.json())
+
     async def _load_resources(self) -> None:
         if self._resources is None:
             response = await self.client.request("GET", "/resources")
             if response.status_code != 200:
                 self._resources = []
                 return
-            self._resources = [
-                ResourceModel(**resource) for resource in response.json()
-            ]
+
+            # Handle both old and new response formats
+            response_data = response.json()
+            if isinstance(response_data, dict) and "resources" in response_data:
+                # Old format: {"resources": [...]}
+                resources_list = response_data["resources"]
+            else:
+                # New format: [...]
+                resources_list = response_data
+
+            self._resources = [ResourceModel(**resource) for resource in resources_list]
             for resource in self._resources:
                 if resource.type not in self._resources_state:
                     self._resources_state[resource.type.value] = {}

fleet/{manager → instance}/models.py

@@ -126,3 +126,16 @@ class Resource(BaseModel):
     type: ResourceType
     mode: ResourceMode
     label: Optional[str] = Field(None, title="Label")
+
+
+class ExecuteFunctionRequest(BaseModel):
+    function_code: str
+    function_name: str
+    text_solution: Optional[str] = None
+
+
+class ExecuteFunctionResponse(BaseModel):
+    success: bool
+    result: Optional[Any] = None
+    error: Optional[str] = None
+    message: str

fleet/playwright.py

@@ -0,0 +1,291 @@
+import base64
+from typing import List, Dict, Any
+from playwright.async_api import async_playwright, Browser, Page
+from .client import AsyncEnvironment
+
+
+# Key mapping for computer use actions
+CUA_KEY_TO_PLAYWRIGHT_KEY = {
+    "/": "Divide",
+    "\\": "Backslash",
+    "alt": "Alt",
+    "arrowdown": "ArrowDown",
+    "arrowleft": "ArrowLeft",
+    "arrowright": "ArrowRight",
+    "arrowup": "ArrowUp",
+    "backspace": "Backspace",
+    "capslock": "CapsLock",
+    "cmd": "Meta",
+    "ctrl": "Control",
+    "delete": "Delete",
+    "end": "End",
+    "enter": "Enter",
+    "esc": "Escape",
+    "home": "Home",
+    "insert": "Insert",
+    "option": "Alt",
+    "pagedown": "PageDown",
+    "pageup": "PageUp",
+    "shift": "Shift",
+    "space": " ",
+    "super": "Meta",
+    "tab": "Tab",
+    "win": "Meta",
+}
+
+
+class FleetPlaywrightWrapper:
+    """
+    A wrapper that adds Playwright browser automation to Fleet environment instances.
+
+    This class handles:
+    - Browser connection via CDP
+    - Computer actions (click, scroll, type, etc.)
+    - Screenshot capture
+    - Integration with OpenAI computer use API
+
+    Usage:
+        instance = await fleet.env.make(env_key="hubspot", version="v1.2.7")
+        browser = FleetPlaywrightWrapper(instance)
+        await browser.start()
+
+        # Use browser methods
+        screenshot = await browser.screenshot()
+        tools = [browser.openai_cua_tool]
+
+        # Clean up when done
+        await browser.close()
+    """
+
+    def get_environment(self):
+        return "browser"
+
+    def get_dimensions(self):
+        return (1920, 1080)
+
+    def __init__(
+        self,
+        env: AsyncEnvironment,
+        display_width: int = 1920,
+        display_height: int = 1080,
+    ):
+        """
+        Initialize the Fleet Playwright wrapper.
+
+        Args:
+            env: Fleet environment instance
+            display_width: Browser viewport width
+            display_height: Browser viewport height
+        """
+        self.env = env
+        self.display_width = display_width
+        self.display_height = display_height
+
+        self._playwright = None
+        self._browser: Browser | None = None
+        self._page: Page | None = None
+        self._started = False
+
+    async def start(self):
+        """Start the browser and establish connection."""
+        if self._started:
+            return
+
+        # Start Playwright
+        self._playwright = await async_playwright().start()
+
+        # Start browser on the Fleet instance
+        print("Starting browser...")
+        await self.env.browser().start()
+        cdp = await self.env.browser().describe()
+
+        # Connect to browser
+        self._browser = await self._playwright.chromium.connect_over_cdp(
+            cdp.cdp_browser_url
+        )
+        self._page = self._browser.contexts[0].pages[0]
+        await self._page.set_viewport_size(
+            {"width": self.display_width, "height": self.display_height}
+        )
+
+        self._started = True
+        print(f"Track agent: {cdp.cdp_devtools_url}")
+
+    async def close(self):
+        """Close the browser connection."""
+        if self._playwright:
+            await self._playwright.stop()
+            self._playwright = None
+            self._browser = None
+            self._page = None
+            self._started = False
+
+    def _ensure_started(self):
+        """Ensure browser is started before operations."""
+        if not self._started:
+            raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    @property
+    def openai_cua_tool(self) -> Dict[str, Any]:
+        """
+        Tool definition for OpenAI computer use API.
+
+        Returns:
+            Tool definition dict for use with OpenAI responses API
+        """
+        return {
+            "type": "computer_use_preview",
+            "display_width": self.display_width,
+            "display_height": self.display_height,
+            "environment": "browser",
+        }
+
+    async def screenshot(self) -> str:
+        """
+        Take a screenshot and return base64 encoded string.
+
+        Returns:
+            Base64 encoded PNG screenshot
+        """
+        self._ensure_started()
+
+        png_bytes = await self._page.screenshot(full_page=False)
+        return base64.b64encode(png_bytes).decode("utf-8")
+
+    def get_current_url(self) -> str:
+        """Get the current page URL."""
+        self._ensure_started()
+        return self._page.url
+
+    async def execute_computer_action(self, action: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Execute a computer action and return the result for OpenAI API.
+
+        Args:
+            action: Computer action dict from OpenAI response
+
+        Returns:
+            Result dict for computer_call_output
+        """
+        self._ensure_started()
+
+        action_type = action["type"]
+        action_args = {k: v for k, v in action.items() if k != "type"}
+
+        print(f"Executing: {action_type}({action_args})")
+
+        # Execute the action
+        if hasattr(self, f"_{action_type}"):
+            method = getattr(self, f"_{action_type}")
+            await method(**action_args)
+        else:
+            raise ValueError(f"Unsupported action type: {action_type}")
+
+        # Take screenshot after action
+        screenshot_base64 = await self.screenshot()
+
+        return {
+            "type": "input_image",
+            "image_url": f"data:image/png;base64,{screenshot_base64}",
+            "current_url": self.get_current_url(),
+        }
+
+    # Computer action implementations
+    async def _click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates."""
+        self._ensure_started()
+        await self._page.mouse.click(x, y, button=button)
+
+    async def _double_click(self, x: int, y: int) -> None:
+        """Double-click at coordinates."""
+        self._ensure_started()
+        await self._page.mouse.dblclick(x, y)
+
+    async def _scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll from coordinates."""
+        self._ensure_started()
+        await self._page.mouse.move(x, y)
+        await self._page.evaluate(f"window.scrollBy({scroll_x}, {scroll_y})")
+
+    async def _type(self, text: str) -> None:
+        """Type text."""
+        self._ensure_started()
+        await self._page.keyboard.type(text)
+
+    async def _keypress(self, keys: List[str]) -> None:
+        """Press key combination."""
+        self._ensure_started()
+        mapped_keys = [CUA_KEY_TO_PLAYWRIGHT_KEY.get(key.lower(), key) for key in keys]
+        for key in mapped_keys:
+            await self._page.keyboard.down(key)
+        for key in reversed(mapped_keys):
+            await self._page.keyboard.up(key)
+
+    async def _move(self, x: int, y: int) -> None:
+        """Move mouse to coordinates."""
+        self._ensure_started()
+        await self._page.mouse.move(x, y)
+
+    async def _drag(self, path: List[Dict[str, int]]) -> None:
+        """Drag mouse along path."""
+        self._ensure_started()
+        if not path:
+            return
+        await self._page.mouse.move(path[0]["x"], path[0]["y"])
+        await self._page.mouse.down()
+        for point in path[1:]:
+            await self._page.mouse.move(point["x"], point["y"])
+        await self._page.mouse.up()
+
+    async def _wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        import asyncio
+
+        await asyncio.sleep(ms / 1000)
+
+    # Browser-specific actions
+    async def _goto(self, url: str) -> None:
+        """Navigate to URL."""
+        self._ensure_started()
+        try:
+            await self._page.goto(url)
+        except Exception as e:
+            print(f"Error navigating to {url}: {e}")
+
+    async def _back(self) -> None:
+        """Go back in browser history."""
+        self._ensure_started()
+        await self._page.go_back()
+
+    async def _forward(self) -> None:
+        """Go forward in browser history."""
+        self._ensure_started()
+        await self._page.go_forward()
+
+    async def _refresh(self) -> None:
+        """Refresh the page."""
+        self._ensure_started()
+        await self._page.reload()
+
+    # ------------------------------------------------------------------
+    # Public aliases (no leading underscore) expected by the Agent &
+    # OpenAI computer-use API. They forward directly to the underscored
+    # implementations above so the external interface matches the older
+    # BasePlaywrightComputer class.
+    # ------------------------------------------------------------------
+
+    # Mouse / keyboard actions
+    click = _click
+    double_click = _double_click
+    scroll = _scroll
+    type = _type  # noqa: A003 – shadowing built-in for API compatibility
+    keypress = _keypress
+    move = _move
+    drag = _drag
+    wait = _wait
+
+    # Browser navigation actions
+    goto = _goto
+    back = _back
+    forward = _forward
+    refresh = _refresh

fleet/resources/base.py

@@ -1,5 +1,5 @@
 from abc import ABC
-from ..manager.models import Resource as ResourceModel, ResourceType, ResourceMode
+from ..instance.models import Resource as ResourceModel, ResourceType, ResourceMode
 
 
 class Resource(ABC):

fleet/resources/browser.py

@@ -1,5 +1,5 @@
 from typing import Optional
-from ..manager.models import (
+from ..instance.models import (
     Resource as ResourceModel,
     CDPDescribeResponse,
     ChromeStartRequest,
@@ -10,14 +10,13 @@ from .base import Resource
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from ..manager.base import AsyncWrapper
+    from ..instance.base import AsyncWrapper
 
 
 class AsyncBrowserResource(Resource):
     def __init__(self, resource: ResourceModel, client: "AsyncWrapper"):
         super().__init__(resource)
         self.client = client
-        self._describe: Optional[CDPDescribeResponse] = None
 
     async def start(self, width: int = 1920, height: int = 1080) -> CDPDescribeResponse:
         response = await self.client.request(
@@ -29,13 +28,11 @@ class AsyncBrowserResource(Resource):
         return await self.describe()
 
     async def describe(self) -> CDPDescribeResponse:
-        if self._describe is None:
+        response = await self.client.request("GET", "/resources/cdp/describe")
+        if response.status_code != 200:
+            await self.start()
             response = await self.client.request("GET", "/resources/cdp/describe")
-            if response.status_code != 200:
-                await self.start()
-                response = await self.client.request("GET", "/resources/cdp/describe")
-            self._describe = CDPDescribeResponse(**response.json())
-        return self._describe
+        return CDPDescribeResponse(**response.json())
 
     async def cdp_url(self) -> str:
         return (await self.describe()).cdp_browser_url

fleet/resources/sqlite.py

@@ -1,12 +1,12 @@
 from typing import Any, List, Optional
-from ..manager.models import Resource as ResourceModel
-from ..manager.models import DescribeResponse, QueryRequest, QueryResponse
+from ..instance.models import Resource as ResourceModel
+from ..instance.models import DescribeResponse, QueryRequest, QueryResponse
 from .base import Resource
 
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from ..manager.base import AsyncWrapper
+    from ..instance.base import AsyncWrapper
 
 
 class AsyncSQLiteResource(Resource):

fleet/verifiers/__init__.py

@@ -1,4 +1,16 @@
-from .database_snapshot import QueryBuilder, DatabaseSnapshot
-from .sql_differ import SQLiteDiffer
+from .db import QueryBuilder, DatabaseSnapshot, SnapshotDiff, IgnoreConfig
+from .code import (
+    TASK_SUCCESSFUL_SCORE,
+    extract_last_assistant_message,
+    execute_validation_function,
+)
 
-__all__ = ["QueryBuilder", "DatabaseSnapshot", "SQLiteDiffer"]
+__all__ = [
+    "DatabaseSnapshot",
+    "QueryBuilder",
+    "SnapshotDiff",
+    "IgnoreConfig",
+    "TASK_SUCCESSFUL_SCORE",
+    "extract_last_assistant_message",
+    "execute_validation_function",
+]

fleet/verifiers/code.py

@@ -0,0 +1,132 @@
+import traceback
+import logging
+from typing import Any, Dict
+from .db import DatabaseSnapshot, IgnoreConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+TASK_SUCCESSFUL_SCORE = 1
+
+
+def extract_last_assistant_message(transcript: str) -> str:
+    """
+    Extract only the last assistant message from the transcript, filtering out tool calls.
+
+    Args:
+        transcript: The full conversation transcript
+
+    Returns:
+        The content of the last assistant message with tool calls filtered out
+    """
+    if not transcript:
+        return ""
+
+    # Split transcript into sections by "Assistant:" markers
+    sections = transcript.split("Assistant:")
+    if len(sections) < 2:
+        # No "Assistant:" markers found, treat entire transcript as assistant message
+        last_assistant_section = transcript
+    else:
+        # Get the last assistant section
+        last_assistant_section = sections[-1]
+
+    # Filter out specific content blocks using regex-like approach
+    import re
+
+    # Remove image blocks: <img src="data:..."/>
+    last_assistant_section = re.sub(
+        r'<img src="data:[^"]*"[^>]*/?>', "", last_assistant_section
+    )
+
+    # Remove tool call blocks:  .../>
+    last_assistant_section = re.sub(
+        r'<tool_call[^>]*>.*?"/>', "", last_assistant_section, flags=re.DOTALL
+    )
+
+    # Remove tool result blocks: <tool_result>...</tool_result>
+    last_assistant_section = re.sub(
+        r"<tool_result>.*?</tool_result>", "", last_assistant_section, flags=re.DOTALL
+    )
+
+    # Clean up extra whitespace
+    filtered_transcript = last_assistant_section.strip()
+
+    return filtered_transcript
+
+
+async def execute_validation_function(
+    function_code: str,
+    function_name: str,
+    before_snapshot_path: str,
+    after_snapshot_path: str,
+    transcript: str | None = None,
+) -> Dict[str, Any]:
+    """
+    Execute arbitrary validation function code with database snapshots.
+
+    Args:
+        function_code: The Python code containing the function definition
+        function_name: Name of the function to call after executing the code
+        before_snapshot_path: Path to the before database snapshot
+        after_snapshot_path: Path to the after database snapshot
+
+    Returns:
+        Dict containing success status, result, and any error message
+    """
+    try:
+        # Create database snapshots
+        before = DatabaseSnapshot(before_snapshot_path)
+        after = DatabaseSnapshot(after_snapshot_path)
+
+        # Create a namespace with the required imports and constants
+        namespace = {
+            "DatabaseSnapshot": DatabaseSnapshot,
+            "IgnoreConfig": IgnoreConfig,
+            "TASK_SUCCESSFUL_SCORE": TASK_SUCCESSFUL_SCORE,
+            "extract_last_assistant_message": extract_last_assistant_message,
+            "__builtins__": __builtins__,
+        }
+
+        # Execute the provided code in the namespace
+        exec(function_code, namespace)
+
+        # Check if the function exists in the namespace
+        if function_name not in namespace:
+            return {
+                "success": False,
+                "error": f"Function '{function_name}' not found in the provided code",
+                "result": None,
+            }
+
+        # Get the function from the namespace
+        func = namespace[function_name]
+
+        # Call the function with before/after snapshots
+        # Support both sync and async functions
+        import inspect
+
+        # Check the function signature to determine how many arguments it accepts
+        sig = inspect.signature(func)
+        param_count = len(sig.parameters)
+
+        if inspect.iscoroutinefunction(func):
+            # Handle async function - we can await it since we're now async
+            if param_count >= 3:
+                result = await func(before, after, transcript)
+            else:
+                result = await func(before, after)
+        else:
+            # Handle sync function
+            if param_count >= 3:
+                result = func(before, after, transcript)
+            else:
+                result = func(before, after)
+
+        return {"success": True, "result": result, "error": None}
+
+    except Exception as e:
+        error_msg = f"Error executing function: {str(e)}\n{traceback.format_exc()}"
+        logger.error(error_msg)
+        return {"success": False, "error": error_msg, "result": None}