fleet-python 0.2.83__py3-none-any.whl → 0.2.85__py3-none-any.whl

fleet/__init__.py

@@ -73,7 +73,7 @@ from . import env
 from . import global_client as _global_client
 from ._async import global_client as _async_global_client
 
-__version__ = "0.2.83"
+__version__ = "0.2.85"
 
 __all__ = [
     # Core classes

fleet/_async/__init__.py

@@ -44,7 +44,7 @@ from ..types import VerifierFunction
 from .. import env
 from . import global_client as _async_global_client
 
-__version__ = "0.2.83"
+__version__ = "0.2.85"
 
 __all__ = [
     # Core classes

fleet/_async/base.py

@@ -26,7 +26,7 @@ from .exceptions import (
 try:
     from .. import __version__
 except ImportError:
-    __version__ = "0.2.83"
+    __version__ = "0.2.85"
 
 logger = logging.getLogger(__name__)
 

fleet/agent/gemini_cua/Dockerfile

@@ -0,0 +1,44 @@
+# MCP Server - Browser control in Docker with optional VNC
+FROM python:3.11-slim
+
+# Install dependencies for Chromium and VNC
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    # Chromium dependencies
+    wget fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 \
+    libatspi2.0-0 libcups2 libdbus-1-3 libdrm2 libgbm1 libgtk-3-0 \
+    libnspr4 libnss3 libxcomposite1 libxdamage1 libxfixes3 libxkbcommon0 \
+    libxrandr2 xdg-utils \
+    # VNC and display for headful mode
+    xvfb x11vnc fluxbox \
+    # noVNC for web-based viewing
+    novnc websockify \
+    # Utilities
+    procps net-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python deps
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt && playwright install chromium
+
+# Copy server files (all from same directory)
+COPY playwright_utils.py .
+COPY mcp_server.py .
+COPY start.sh .
+RUN chmod +x start.sh
+
+# Environment
+ENV PORT=8765 \
+    SCREEN_WIDTH=1366 \
+    SCREEN_HEIGHT=768 \
+    HEADLESS=true \
+    VNC_PORT=5900 \
+    NOVNC_PORT=6080 \
+    DISPLAY=:99
+
+# Expose ports: MCP server, VNC, noVNC
+EXPOSE 8765 5900 6080
+
+# Start script handles display setup
+CMD ["./start.sh"]

fleet/agent/gemini_cua/agent.py

@@ -25,6 +25,7 @@ from mcp import ClientSession
 from mcp.client.streamable_http import streamable_http_client
 from google import genai
 from google.genai import types
+import fleet
 from fleet.utils.logging import log_verbose, VERBOSE
 
 # Whitelist hooks for auto-detecting model endpoints (optional)
@@ -136,20 +137,36 @@ class MCP:
         result = await self._session.call_tool(name, args or {})
         duration_ms = int((time.time() - start_time) * 1000)
         
+        # Debug: log raw MCP result structure
+        log_verbose(f"    MCP result.content ({len(result.content)} items):")
+        for i, item in enumerate(result.content):
+            log_verbose(f"      [{i}] type={type(item).__name__}, attrs={dir(item)[:10]}...")
+            if hasattr(item, "type"):
+                log_verbose(f"          .type = {repr(item.type)}")
+            if hasattr(item, "data"):
+                data_preview = str(item.data)[:50] if item.data else "None"
+                log_verbose(f"          .data = {data_preview}...")
+        
+        # Helper to get attribute or dict key
+        def _get(item, key, default=None):
+            if isinstance(item, dict):
+                return item.get(key, default)
+            return getattr(item, key, default)
+        
         # Convert MCP result to dict format expected by agent
         content = []
         for item in result.content:
-            if hasattr(item, "type"):
-                if item.type == "image":
-                    content.append({
-                        "type": "image",
-                        "data": item.data[:100] + "..." if len(item.data) > 100 else item.data,  # Truncate for logging
-                        "mimeType": getattr(item, "mimeType", "image/png"),
-                    })
-                elif item.type == "text":
-                    content.append({"type": "text", "text": item.text})
+            item_type = _get(item, "type")
+            if item_type == "image":
+                content.append({
+                    "type": "image",
+                    "data": _get(item, "data", ""),
+                    "mimeType": _get(item, "mimeType", "image/png"),
+                })
+            elif item_type == "text":
+                content.append({"type": "text", "text": _get(item, "text", "")})
         
-        # Log the call
+        # Log the call (just types, not data)
         self._log({
             "type": "mcp_call",
             "tool": name,
@@ -158,20 +175,7 @@ class MCP:
             "response_content_types": [c.get("type") for c in content],
             "is_error": result.isError if hasattr(result, "isError") else False,
         })
-        
-        # Return full content (not truncated)
-        full_content = []
-        for item in result.content:
-            if hasattr(item, "type"):
-                if item.type == "image":
-                    full_content.append({
-                        "type": "image",
-                        "data": item.data,
-                        "mimeType": getattr(item, "mimeType", "image/png"),
-                    })
-                elif item.type == "text":
-                    full_content.append({"type": "text", "text": item.text})
-        return {"content": full_content, "isError": result.isError if hasattr(result, "isError") else False}
+        return {"content": content, "isError": result.isError if hasattr(result, "isError") else False}
     
     def get_tools(self) -> List[Dict]:
         """Return the list of tools from the server."""
@@ -201,12 +205,13 @@ def get_image_data(result: Dict) -> Optional[str]:
 class GeminiAgent:
     """Gemini Computer Use Agent."""
     
-    def __init__(self, mcp: MCP, model: str):
+    def __init__(self, mcp: MCP, model: str, session=None):
         self.mcp = mcp
         # Strip provider prefix if present
         self.model = model.split("/")[-1] if "/" in model else model
         self.client = get_gemini_client()
         self.transcript: List[Dict] = []
+        self.session = session  # Fleet session for live logging
     
     async def _execute_tool(self, name: str, args: Dict) -> Dict:
         return await self.mcp.call(name, args)
@@ -251,8 +256,13 @@ STRICT RULES:
             max_output_tokens=4096,
             system_instruction=system_prompt,
             tools=[types.Tool(function_declarations=gemini_tools)],
+            thinking_config=types.ThinkingConfig(include_thoughts=True),
         )
         
+        # Set config on session for logging (if session exists)
+        if self.session:
+            self.session.config = config
+        
         history: List[types.Content] = []
         
         user_prompt = f"""###User instruction: {prompt}"""
@@ -292,6 +302,15 @@ STRICT RULES:
                 log_verbose(f"  Candidate: {candidate}")
                 continue
             
+            # Log to Fleet session (live)
+            if self.session:
+                try:
+                    await self.session.log(history, response)
+                    if step == 1 and self.session.session_id:
+                        print(f"Session: https://fleetai.com/dashboard/sessions/{self.session.session_id}")
+                except Exception as e:
+                    log_verbose(f"  [WARN] Session log failed: {e}")
+            
             # Log all parts for debugging
             log_verbose(f"\n  Response parts ({len(candidate.content.parts)}):")
             for i, part in enumerate(candidate.content.parts):
@@ -415,6 +434,8 @@ async def main():
         "url": os.environ.get("FLEET_MCP_URL", "http://localhost:8765"),
         "prompt": os.environ.get("FLEET_TASK_PROMPT", ""),
         "task_key": os.environ.get("FLEET_TASK_KEY", ""),
+        "job_id": os.environ.get("FLEET_JOB_ID"),
+        "instance_id": os.environ.get("FLEET_INSTANCE_ID"),
         "model": os.environ.get("FLEET_MODEL", "gemini-2.5-pro"),
         "max_steps": int(os.environ.get("FLEET_MAX_STEPS", "100")),
     }
@@ -430,10 +451,24 @@ async def main():
         print(json.dumps(result))
         return result
     
+    # Create Fleet session for live logging
+    session = None
+    if os.environ.get("FLEET_API_KEY"):
+        session = fleet.session_async(
+            job_id=config["job_id"],
+            model=config["model"],
+            task_key=config["task_key"],
+            instance_id=config["instance_id"],
+        )
+    
     async with MCP(config["url"]) as mcp:
-        agent = GeminiAgent(mcp, config["model"])
+        agent = GeminiAgent(mcp, config["model"], session=session)
         result = await agent.run(config["prompt"], config["max_steps"])
         result["task_key"] = config["task_key"]
+        # Include session_id in result so orchestrator can complete it after verification
+        if session and session.session_id:
+            result["session_id"] = session.session_id
+        
         print(json.dumps(result))
         return result
 

fleet/agent/gemini_cua/mcp_server.py

@@ -18,6 +18,7 @@ from contextlib import asynccontextmanager
 from typing import Optional
 
 from mcp.server.fastmcp import FastMCP
+from mcp.types import ImageContent, TextContent
 from starlette.requests import Request
 from starlette.responses import JSONResponse
 
@@ -227,9 +228,10 @@ def _dy(y: int) -> int:
 
 
 def _screenshot_response(img: bytes) -> list:
+    """Return screenshot as proper MCP content types."""
     return [
-        {"type": "image", "data": base64.b64encode(img).decode(), "mimeType": "image/png"},
-        {"type": "text", "text": f"URL: {computer.current_url}"}
+        ImageContent(type="image", data=base64.b64encode(img).decode(), mimeType="image/png"),
+        TextContent(type="text", text=f"URL: {computer.current_url}"),
     ]
 
 

fleet/agent/gemini_cua/playwright_utils.py

@@ -0,0 +1,440 @@
+"""Playwright browser control utilities.
+
+Provides PlaywrightComputer class for browser automation with:
+- Mouse actions (click, move, drag, scroll)
+- Keyboard actions (type, key combinations)
+- Screenshot capture
+- Normalized coordinate support (0-1000 range)
+
+Key mapping follows the action spec convention for cross-platform compatibility.
+"""
+
+import asyncio
+import logging
+from typing import List, Optional, Tuple
+
+from playwright.async_api import async_playwright, Page, Browser, BrowserContext
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Key Mapping - Action spec keys to Playwright keys
+# =============================================================================
+
+PLAYWRIGHT_KEY_MAP = {
+    # Common keys
+    "enter": "Enter", "return": "Enter", "tab": "Tab",
+    "escape": "Escape", "esc": "Escape", "space": " ",
+    "backspace": "Backspace", "delete": "Delete", "insert": "Insert",
+    
+    # Modifiers
+    "alt": "Alt", "alt_left": "Alt", "alt_right": "Alt",
+    "control": "Control", "control_left": "Control", "control_right": "Control",
+    "ctrl": "Control", "ctrl_left": "Control", "ctrl_right": "Control",
+    "shift": "Shift", "shift_left": "Shift", "shift_right": "Shift",
+    "caps_lock": "CapsLock", "capslock": "CapsLock",
+    "meta": "Meta", "meta_left": "Meta", "meta_right": "Meta",
+    "command": "Meta", "cmd": "Meta", "super": "Meta", "win": "Meta", "windows": "Meta",
+    "num_lock": "NumLock", "numlock": "NumLock",
+    "scroll_lock": "ScrollLock", "scrolllock": "ScrollLock",
+    
+    # Navigation
+    "arrow_down": "ArrowDown", "arrow_up": "ArrowUp",
+    "arrow_left": "ArrowLeft", "arrow_right": "ArrowRight",
+    "down": "ArrowDown", "up": "ArrowUp", "left": "ArrowLeft", "right": "ArrowRight",
+    "end": "End", "home": "Home",
+    "page_down": "PageDown", "pagedown": "PageDown",
+    "page_up": "PageUp", "pageup": "PageUp",
+    
+    # Function keys
+    **{f"f{i}": f"F{i}" for i in range(1, 21)},
+    
+    # Symbols
+    "backquote": "`", "grave": "`", "tilde": "`",
+    "backslash": "\\", "bracket_left": "[", "bracketleft": "[",
+    "bracket_right": "]", "bracketright": "]",
+    "comma": ",", "double_quote": '"', "doublequote": '"',
+    "equal": "=", "equals": "=", "minus": "-", "dash": "-",
+    "period": ".", "dot": ".", "quote": "'", "apostrophe": "'",
+    "semicolon": ";", "slash": "/", "forward_slash": "/", "forwardslash": "/",
+    
+    # Numpad
+    **{f"numpad_{i}": f"Numpad{i}" for i in range(10)},
+    **{f"numpad{i}": f"Numpad{i}" for i in range(10)},
+    "numpad_add": "NumpadAdd", "numpadadd": "NumpadAdd",
+    "numpad_subtract": "NumpadSubtract", "numpadsubtract": "NumpadSubtract",
+    "numpad_multiply": "NumpadMultiply", "numpadmultiply": "NumpadMultiply",
+    "numpad_divide": "NumpadDivide", "numpaddivide": "NumpadDivide",
+    "numpad_decimal": "NumpadDecimal", "numpaddecimal": "NumpadDecimal",
+    "numpad_enter": "NumpadEnter", "numpadenter": "NumpadEnter",
+    
+    # Media
+    "audio_volume_mute": "AudioVolumeMute",
+    "audio_volume_down": "AudioVolumeDown",
+    "audio_volume_up": "AudioVolumeUp",
+    "media_track_next": "MediaTrackNext",
+    "media_track_previous": "MediaTrackPrevious",
+    "media_stop": "MediaStop",
+    "media_play_pause": "MediaPlayPause",
+    
+    # Other
+    "print_screen": "PrintScreen", "printscreen": "PrintScreen",
+    "pause": "Pause", "context_menu": "ContextMenu", "contextmenu": "ContextMenu",
+    "help": "Help",
+}
+
+MODIFIER_KEYS = {
+    "Alt", "Control", "Shift", "Meta",
+    "alt", "alt_left", "alt_right",
+    "control", "control_left", "control_right", "ctrl", "ctrl_left", "ctrl_right",
+    "shift", "shift_left", "shift_right",
+    "meta", "meta_left", "meta_right", "command", "cmd", "super", "win", "windows",
+}
+
+# Key specification for tool docstrings
+KEY_SPEC = (
+    "Key specification: * Common: enter, tab, escape, space, backspace, delete "
+    "* Modifiers: alt_left, control_left, control_right, shift_left, caps_lock, meta "
+    "* Navigation: arrow_down, arrow_right, end, home, page_down "
+    "* Function: f1 to f12 "
+    "* Alphanumeric: key_a to key_z, digit_0 to digit_9 "
+    "* Symbols: backquote, backslash, bracket_left, bracket_right, comma, double_quote, "
+    "equal, minus, period, quote, semicolon, slash "
+    "* Numpad: numpad_0 to numpad_9, numpad_add, numpad_divide, numpad_enter, numpad_multiply"
+)
+
+
+def map_key(key: str) -> str:
+    """Map action spec key name to Playwright key name.
+    
+    Args:
+        key: Key name in action spec format (e.g., "key_a", "control_left")
+        
+    Returns:
+        Playwright key name (e.g., "a", "Control")
+    """
+    k = key.lower().strip()
+    if k in PLAYWRIGHT_KEY_MAP:
+        return PLAYWRIGHT_KEY_MAP[k]
+    if k.startswith("key_") and len(k) == 5:
+        return k[4].lower()
+    if k.startswith("digit_") and len(k) == 7:
+        return k[6]
+    if len(key) == 1:
+        return key
+    return key
+
+
+def is_modifier(key: str) -> bool:
+    """Check if a key is a modifier key.
+    
+    Args:
+        key: Key name to check
+        
+    Returns:
+        True if the key is a modifier (Alt, Control, Shift, Meta)
+    """
+    return key.lower().strip() in MODIFIER_KEYS or map_key(key) in {"Alt", "Control", "Shift", "Meta"}
+
+
+# =============================================================================
+# PlaywrightComputer - Browser control
+# =============================================================================
+
+class PlaywrightComputer:
+    """Browser control via Playwright.
+    
+    Provides a high-level interface for browser automation:
+    - Mouse actions with optional visual highlighting
+    - Keyboard input with proper modifier handling
+    - Screenshot capture
+    - Automatic page load waiting
+    
+    Args:
+        screen_size: Tuple of (width, height) for viewport
+        initial_url: URL to navigate to on start
+        headless: Run browser without visible window
+        highlight_mouse: Show visual indicator for mouse actions (useful for debugging)
+    
+    Example:
+        computer = PlaywrightComputer(
+            screen_size=(1366, 768),
+            initial_url="https://example.com",
+            headless=False,
+            highlight_mouse=True,
+        )
+        await computer.start()
+        await computer.mouse_click(683, 384)  # Click center
+        screenshot = await computer.screenshot()
+        await computer.stop()
+    """
+    
+    def __init__(
+        self,
+        screen_size: Tuple[int, int],
+        initial_url: str,
+        headless: bool = True,
+        highlight_mouse: bool = False,
+    ):
+        self._screen_size = screen_size
+        self._initial_url = initial_url
+        self._headless = headless
+        self._highlight_mouse = highlight_mouse
+        self._playwright = None
+        self._browser: Optional[Browser] = None
+        self._context: Optional[BrowserContext] = None
+        self._page: Optional[Page] = None
+    
+    @property
+    def width(self) -> int:
+        """Viewport width in pixels."""
+        return self._screen_size[0]
+    
+    @property
+    def height(self) -> int:
+        """Viewport height in pixels."""
+        return self._screen_size[1]
+    
+    @property
+    def current_url(self) -> str:
+        """Current page URL."""
+        return self._page.url if self._page else ""
+    
+    async def _handle_new_page(self, new_page: Page):
+        """Handle new tab by redirecting to current page."""
+        new_url = new_page.url
+        await new_page.close()
+        await self._page.goto(new_url)
+    
+    async def start(self):
+        """Start the browser and navigate to initial URL."""
+        logger.info(f"Starting browser (headless={self._headless})...")
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(
+            headless=self._headless,
+            args=[
+                "--no-sandbox",
+                "--disable-extensions",
+                "--disable-file-system",
+                "--disable-plugins",
+                "--disable-dev-shm-usage",
+                "--disable-background-networking",
+                "--disable-default-apps",
+                "--disable-sync",
+            ],
+        )
+        self._context = await self._browser.new_context(
+            viewport={"width": self._screen_size[0], "height": self._screen_size[1]}
+        )
+        self._page = await self._context.new_page()
+        self._context.on("page", self._handle_new_page)
+        await self._page.goto(self._initial_url)
+        await self._page.wait_for_load_state()
+        logger.info(f"Browser ready: {self._initial_url}")
+    
+    async def stop(self):
+        """Stop the browser and clean up resources."""
+        if self._context:
+            await self._context.close()
+        if self._browser:
+            try:
+                await self._browser.close()
+            except Exception:
+                pass
+        if self._playwright:
+            await self._playwright.stop()
+        logger.info("Browser stopped")
+    
+    async def screenshot(self) -> bytes:
+        """Take a screenshot of the current viewport.
+        
+        Returns:
+            PNG image data as bytes
+        """
+        await self._page.wait_for_load_state()
+        await asyncio.sleep(0.5)
+        return await self._page.screenshot(type="png", full_page=False)
+    
+    async def _highlight(self, x: int, y: int):
+        """Show visual highlight at mouse position (for debugging)."""
+        if not self._highlight_mouse:
+            return
+        await self._page.evaluate(f"""
+            () => {{
+                const div = document.createElement('div');
+                div.style.cssText = 'position:fixed;width:20px;height:20px;border-radius:50%;border:4px solid red;pointer-events:none;z-index:9999;left:{x-10}px;top:{y-10}px;';
+                document.body.appendChild(div);
+                setTimeout(() => div.remove(), 2000);
+            }}
+        """)
+        await asyncio.sleep(1)
+    
+    # -------------------------------------------------------------------------
+    # Mouse actions
+    # -------------------------------------------------------------------------
+    
+    async def mouse_click(self, x: int, y: int, button: str = "left", repeats: int = 1) -> None:
+        """Click at position.
+        
+        Args:
+            x: X coordinate in pixels
+            y: Y coordinate in pixels
+            button: Mouse button ('left', 'middle', 'right')
+            repeats: Number of clicks (2 for double-click)
+        """
+        await self._highlight(x, y)
+        for _ in range(repeats):
+            await self._page.mouse.click(x, y, button=button)
+        await self._page.wait_for_load_state()
+    
+    async def mouse_move(self, x: int, y: int) -> None:
+        """Move mouse to position.
+        
+        Args:
+            x: X coordinate in pixels
+            y: Y coordinate in pixels
+        """
+        await self._highlight(x, y)
+        await self._page.mouse.move(x, y)
+        await self._page.wait_for_load_state()
+    
+    async def mouse_down(self, button: str = "left") -> None:
+        """Press mouse button down.
+        
+        Args:
+            button: Mouse button ('left', 'middle', 'right')
+        """
+        await self._page.mouse.down(button=button)
+    
+    async def mouse_up(self, button: str = "left") -> None:
+        """Release mouse button.
+        
+        Args:
+            button: Mouse button ('left', 'middle', 'right')
+        """
+        await self._page.mouse.up(button=button)
+        await self._page.wait_for_load_state()
+    
+    async def mouse_scroll(self, dx: int, dy: int) -> None:
+        """Scroll the mouse wheel.
+        
+        Args:
+            dx: Horizontal scroll amount in pixels
+            dy: Vertical scroll amount in pixels
+        """
+        await self._page.mouse.wheel(dx, dy)
+        await self._page.wait_for_load_state()
+    
+    async def mouse_drag(
+        self,
+        x_start: int,
+        y_start: int,
+        x_end: int,
+        y_end: int,
+        button: str = "left",
+    ) -> None:
+        """Drag from one position to another.
+        
+        Args:
+            x_start: Starting X coordinate
+            y_start: Starting Y coordinate
+            x_end: Ending X coordinate
+            y_end: Ending Y coordinate
+            button: Mouse button to hold during drag
+        """
+        await self._highlight(x_start, y_start)
+        await self._page.mouse.move(x_start, y_start)
+        await self._page.mouse.down(button=button)
+        await self._highlight(x_end, y_end)
+        await self._page.mouse.move(x_end, y_end)
+        await self._page.mouse.up(button=button)
+        await self._page.wait_for_load_state()
+    
+    # -------------------------------------------------------------------------
+    # Keyboard actions
+    # -------------------------------------------------------------------------
+    
+    async def type_text(self, text: str, press_enter: bool = False) -> None:
+        """Type text using the keyboard.
+        
+        Args:
+            text: Text to type
+            press_enter: Whether to press Enter after typing
+        """
+        await self._page.keyboard.type(text)
+        await self._page.wait_for_load_state()
+        if press_enter:
+            await self._page.keyboard.press("Enter")
+            await self._page.wait_for_load_state()
+    
+    async def key_combination(self, keys: List[str]) -> None:
+        """Press a key combination (e.g., Ctrl+C).
+        
+        Handles modifiers properly - holds them down while pressing other keys.
+        
+        Args:
+            keys: List of keys to press together
+        """
+        if not keys:
+            return
+        
+        modifiers = [map_key(k) for k in keys if is_modifier(k)]
+        regular = [map_key(k) for k in keys if not is_modifier(k)]
+        
+        # Press modifiers down
+        for mod in modifiers:
+            await self._page.keyboard.down(mod)
+        
+        # Press regular keys
+        for key in regular:
+            await self._page.keyboard.press(key)
+        
+        # If only modifiers, brief pause
+        if not regular and modifiers:
+            await asyncio.sleep(0.05)
+        
+        # Release modifiers
+        for mod in reversed(modifiers):
+            await self._page.keyboard.up(mod)
+        
+        await self._page.wait_for_load_state()
+    
+    async def key_down(self, key: str) -> None:
+        """Press a key down (without releasing).
+        
+        Args:
+            key: Key to press down
+        """
+        await self._page.keyboard.down(map_key(key))
+    
+    async def key_up(self, key: str) -> None:
+        """Release a key.
+        
+        Args:
+            key: Key to release
+        """
+        await self._page.keyboard.up(map_key(key))
+        await self._page.wait_for_load_state()
+    
+    # -------------------------------------------------------------------------
+    # Utilities
+    # -------------------------------------------------------------------------
+    
+    async def wait(self, seconds: int) -> None:
+        """Wait for a number of seconds.
+        
+        Args:
+            seconds: Number of seconds to wait
+        """
+        await asyncio.sleep(seconds)
+    
+    async def goto(self, url: str) -> None:
+        """Navigate to a URL.
+        
+        Args:
+            url: URL to navigate to
+        """
+        await self._page.goto(url)
+        await self._page.wait_for_load_state()
+

fleet/agent/gemini_cua/requirements.txt

@@ -0,0 +1,4 @@
+playwright>=1.40.0
+mcp[cli]>=1.2.0
+uvicorn>=0.30.0
+starlette>=0.38.0

fleet/agent/gemini_cua/start.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+# Start virtual display if not headless
+if [ "$HEADLESS" != "true" ]; then
+    echo "Starting Xvfb virtual display..."
+    Xvfb :99 -screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x24 &
+    sleep 1
+    
+    echo "Starting fluxbox window manager..."
+    fluxbox &
+    sleep 1
+    
+    echo "Starting VNC server on port $VNC_PORT..."
+    x11vnc -display :99 -forever -shared -rfbport $VNC_PORT -nopw &
+    sleep 1
+    
+    echo "Starting noVNC on port $NOVNC_PORT..."
+    websockify --web=/usr/share/novnc/ $NOVNC_PORT localhost:$VNC_PORT &
+    sleep 1
+    
+    echo ""
+    echo "=========================================="
+    echo "  Browser visible at: http://localhost:$NOVNC_PORT/vnc.html"
+    echo "=========================================="
+    echo ""
+fi
+
+# Start the MCP server
+exec python mcp_server.py
+

fleet/agent/orchestrator.py

@@ -21,15 +21,14 @@ import asyncio
 import json
 import logging
 import os
-import subprocess
 import time
+from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
+import fleet
 from .utils import get_agent_path
 from .types import AgentConfig, AgentResult, TaskResult
-from fleet.proxy import ProxyManager
-from fleet.eval import TrafficUploader
 
 logger = logging.getLogger(__name__)
 
@@ -45,11 +44,6 @@ class AgentOrchestrator:
         self._docker_image: Optional[str] = None
         # Track available ports (recycled when tasks complete)
         self._available_ports: List[Tuple[int, int]] = []
-        # MITM proxy for traffic capture
-        self._proxy: Optional[ProxyManager] = None
-        self._proxy_env: Dict[str, str] = {}
-        # Traffic uploader (tails proxy log, ships to backend)
-        self._uploader: Optional[TrafficUploader] = None
     
     async def _get_next_ports(self) -> Tuple[int, int]:
         """Get next available MCP port and VNC port."""
@@ -75,38 +69,18 @@ class AgentOrchestrator:
         from rich.console import Console
         from rich.live import Live
         from rich.spinner import Spinner
-        import uuid
         
         console = Console()
         
-        # Generate job ID for this run
-        self._job_id = f"eval_{uuid.uuid4().hex[:12]}"
-        console.print(f"Eval job: {self._job_id}")
+        # Create job via Fleet API
+        job_name = f"eval-{self.config.agent}-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        self._job_id = await fleet.job_async(name=job_name)
+        console.print(f"Job: https://fleetai.com/dashboard/jobs/{self._job_id}")
         
         # Create log directory: ~/.fleet/logs/{job_id}/
         self._log_dir = Path.home() / ".fleet" / "logs" / self._job_id
         self._log_dir.mkdir(parents=True, exist_ok=True)
         
-        # Start MITM proxy for traffic capture
-        self._proxy = ProxyManager()
-        try:
-            self._proxy_env = await self._proxy.start()
-            console.print(f"Proxy started, logging to: {self._proxy.log_path}")
-            
-            # Start traffic uploader (tails proxy log, ships raw to backend)
-            self._uploader = TrafficUploader(
-                job_id=self._job_id,
-                log_file=self._proxy.log_path,
-                whitelist=None,  # No filter - upload everything
-            )
-            await self._uploader.start()
-        except Exception as e:
-            console.print(f"[yellow]⚠[/yellow] Proxy failed to start: {e}")
-            console.print("[dim]  Proxy requires aiohttp: pip install aiohttp[/dim]")
-            self._proxy = None
-            self._proxy_env = {}
-            self._uploader = None
-        
         # Load tasks with spinner
         with Live(Spinner("dots", text=f"Loading tasks from {self.config.project_key}..."), console=console, transient=True):
             if self.config.task_keys:
@@ -168,16 +142,6 @@ class AgentOrchestrator:
             else:
                 final.append(r)
         
-        # Stop uploader first (flushes remaining entries)
-        if self._uploader:
-            await self._uploader.stop()
-            stats = self._uploader.stats
-            console.print(f"Traffic: {stats['read']} read, {stats['uploaded']} uploaded")
-        
-        # Stop proxy
-        if self._proxy:
-            await self._proxy.stop()
-        
         # Show logs location
         if hasattr(self, '_log_dir') and self._log_dir.exists():
             session_logs = list(self._log_dir.glob("*.jsonl"))
@@ -198,17 +162,12 @@ class AgentOrchestrator:
         
         image_name = f"fleet-cua-{agent_path.name}"
         
-        # Use fleet SDK root as build context (so Dockerfile can access fleet/utils)
-        # agent_path is like: .../fleet-sdk/fleet/agent/gemini_cua
-        # We want: .../fleet-sdk
-        fleet_root = agent_path.parent.parent.parent
-        
+        # Build context is the agent directory (all files are self-contained)
         with Live(Spinner("dots", text=f"Building Docker image {image_name}..."), console=console, transient=True):
             proc = await asyncio.create_subprocess_exec(
                 "docker", "build",
                 "-t", image_name,
-                "-f", str(dockerfile),
-                str(fleet_root),  # Build context is repo root
+                str(agent_path),  # Build context is agent directory
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
             )
@@ -280,12 +239,14 @@ class AgentOrchestrator:
                 port=port,
                 task_prompt=task_prompt,
                 task_key=task_key,
+                instance_id=env.instance_id,
             )
             logger.debug(f"[{short_key}] Agent done: completed={agent_result.completed}")
             
             # 4. Run verification
             verification_success = None
             verification_score = None
+            verifier_execution_id = None
             
             if agent_result.completed and task.verifier:
                 logger.info(f"[{task_key}] Running verification...")
@@ -295,12 +256,27 @@ class AgentOrchestrator:
                         final_answer=agent_result.final_answer,
                     )
                     verification_success = v.success
+                    verifier_execution_id = v.execution_id
                     # Score is in v.result (the verifier function's return value)
                     verification_score = v.result if isinstance(v.result, (int, float)) else None
                     logger.info(f"[{task_key}] Verification: {verification_success}")
                 except Exception as e:
                     logger.error(f"[{task_key}] Verification error: {e}")
             
+            # 5. Complete/fail session (session was created by agent, we just complete it)
+            session_id = getattr(agent_result, 'session_id', None)
+            if session_id:
+                try:
+                    # Create session object to complete it
+                    session = fleet.session_async(session_id=session_id)
+                    if verification_success:
+                        await session.complete(verifier_execution_id=verifier_execution_id)
+                    else:
+                        await session.fail(verifier_execution_id=verifier_execution_id)
+                    logger.info(f"[{task_key}] Session: https://fleetai.com/dashboard/sessions/{session_id}")
+                except Exception as e:
+                    logger.error(f"[{task_key}] Session complete error: {e}")
+            
             return TaskResult(
                 task_key=task_key,
                 task_prompt=task_prompt,
@@ -414,6 +390,7 @@ class AgentOrchestrator:
         port: int,
         task_prompt: str,
         task_key: str,
+        instance_id: Optional[str] = None,
     ) -> AgentResult:
         """Run agent process."""
         agent_path = get_agent_path(self.config.agent)
@@ -431,6 +408,7 @@ class AgentOrchestrator:
             "FLEET_JOB_ID": self._job_id,
             "FLEET_TASK_PROMPT": task_prompt,
             "FLEET_TASK_KEY": task_key,
+            "FLEET_INSTANCE_ID": instance_id or "",
             "FLEET_MODEL": self.config.model,
             "FLEET_MAX_STEPS": str(self.config.max_steps),
             "FLEET_SCREEN_WIDTH": str(self.config.screen_width),
@@ -438,8 +416,6 @@ class AgentOrchestrator:
             "FLEET_VERBOSE": "true" if self.config.verbose else "false",
         })
         env.update(self.config.api_keys)
-        # Add proxy env vars for traffic capture
-        env.update(self._proxy_env)
         
         proc = await asyncio.create_subprocess_exec(
             "python", str(agent_script),
@@ -494,6 +470,7 @@ class AgentOrchestrator:
                 steps_taken=result_json.get("steps_taken", 0),
                 execution_time_ms=result_json.get("execution_time_ms", 0),
                 transcript=result_json.get("transcript", []),
+                session_id=result_json.get("session_id"),
             )
         
         # Include stderr in error message

fleet/agent/types.py

@@ -33,6 +33,7 @@ class AgentResult(BaseModel):
     steps_taken: int = 0
     execution_time_ms: int = 0
     transcript: List[Dict[str, Any]] = Field(default_factory=list)
+    session_id: Optional[str] = None  # Fleet session ID for completion
 
 
 class TaskResult(BaseModel):

fleet/base.py

@@ -27,7 +27,7 @@ from .exceptions import (
 try:
     from . import __version__
 except ImportError:
-    __version__ = "0.2.83"
+    __version__ = "0.2.85"
 
 logger = logging.getLogger(__name__)
 

{fleet_python-0.2.83.dist-info → fleet_python-0.2.85.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fleet-python
-Version: 0.2.83
+Version: 0.2.85
 Summary: Python SDK for Fleet environments
 Author-email: Fleet AI <nic@fleet.so>
 License: Apache-2.0

{fleet_python-0.2.83.dist-info → fleet_python-0.2.85.dist-info}/RECORD

@@ -23,8 +23,8 @@ examples/openai_simple_example.py,sha256=HmiufucrAZne7tHq9uoEsDWlEhjNC265bQAyIGB
 examples/query_builder_example.py,sha256=-cOMfWGNifYfYEt_Ds73XpwATZvFDL6F4KTkVxdMjzg,3951
 examples/quickstart.py,sha256=1VT39IRRhemsJgxi0O0gprdpcw7HB4pYO97GAYagIcg,3788
 examples/test_cdp_logging.py,sha256=AkCwQCgOTQEI8w3v0knWK_4eXMph7L9x07wj9yIYM10,2836
-fleet/__init__.py,sha256=HoKk1ZMCG5h9tFo0Fg0Akpcnc5UyoMe-8gTr4cWBcFA,7683
-fleet/base.py,sha256=0ekxdFfuhjP6UGQTfoOBffuU_qfKASyJj2KkeBA7BCw,10065
+fleet/__init__.py,sha256=COtmFIP-72TCZB5y0jwNy_PbHuaXY3Sem7UREJbi554,7683
+fleet/base.py,sha256=Lo1tHYRlGuXYNA547HTz-psjuawHbOFxJAI05JHQYgk,10065
 fleet/cli.py,sha256=kXtinWio3hniLq8CyBCwmy6YYgX3yovx1HcqPO2ERdc,36025
 fleet/client.py,sha256=SuNf5IcU8UNDlH-kIe-RlyJ32fBlbxpELBy-43xrrvE,67043
 fleet/config.py,sha256=n_wh9Sahu3gGE7nHJ7kqNFUH1qDiBtF4bgZq9MvIBMU,319
@@ -33,8 +33,8 @@ fleet/global_client.py,sha256=frrDAFNM2ywN0JHLtlm9qbE1dQpnQJsavJpb7xSR_bU,1072
 fleet/models.py,sha256=Q-bGdXBazExADtvKTpMOYo77poizXol4wtGLH3tlj08,25111
 fleet/tasks.py,sha256=8pEzXmgC7RslqsMC_0s6shhr_t2WGIRpTRqo-MAQjdg,20778
 fleet/types.py,sha256=L4Y82xICf1tzyCLqhLYUgEoaIIS5h9T05TyFNHSWs3s,652
-fleet/_async/__init__.py,sha256=Y8dtXVAM-VC-wkmX8NRnUSUeytO0S0mfHTbYpZpdQIU,9115
-fleet/_async/base.py,sha256=mdFxKJGTY_r3WnyxgRD-8WBtLusu457wWHweFsBHCd0,9630
+fleet/_async/__init__.py,sha256=DdZeU7jd0L-li1bynFCnkh0037Wxl623w1sUWoXdz68,9115
+fleet/_async/base.py,sha256=X73TI7gLUXbks6Abe9FnoCivRMJqGdPEMjg5QHpqfzI,9630
 fleet/_async/client.py,sha256=8UhY61h1M6iSb1WpncmHScLEe-zTTDCABDG11apaKGU,63451
 fleet/_async/exceptions.py,sha256=fUmPwWhnT8SR97lYsRq0kLHQHKtSh2eJS0VQ2caSzEI,5055
 fleet/_async/global_client.py,sha256=4WskpLHbsDEgWW7hXMD09W-brkp4euy8w2ZJ88594rQ,1103
@@ -54,12 +54,16 @@ fleet/_async/verifiers/__init__.py,sha256=1WTlCNq4tIFbbXaQu5Bf2WppZq0A8suhtZbxMT
 fleet/_async/verifiers/bundler.py,sha256=9aWWXFsovBPcndE06IATn5jaeli5fRORAYeenF9heN0,26264
 fleet/_async/verifiers/verifier.py,sha256=iSa-rO-E1R3IQTFS9Z7jbQvQVtsDkilITQP9IIQU2JA,14556
 fleet/agent/__init__.py,sha256=BuiElLoL_OTq_tmEMD85Zc1x3x9iUerYM7TxLpD22aI,737
-fleet/agent/orchestrator.py,sha256=HRbeBgXgX9XgJXNboFGjWkhmoZc8ZRJNesCXvzS9mXY,20375
-fleet/agent/types.py,sha256=Ms884KhHdbYOMTwf8uWaBWe3FAkKU_w5YAt5g0Yfky4,1404
+fleet/agent/orchestrator.py,sha256=cUJIrrfanbmqgVcZB6-KNngOT4Mr_vmNGbLylBH9wmY,19777
+fleet/agent/types.py,sha256=v9EpttwyOBDwwMo0lABQmIvT3xlRQqfqLkHXJjsOmrw,1476
 fleet/agent/utils.py,sha256=VNhyIFwTKizl00ccqXdhh6s3DYZxPy55PMXgYhTLNX4,1171
+fleet/agent/gemini_cua/Dockerfile,sha256=uirE7L7JzmHpCTHTrIAsPg-LTdhlr78ajqqBJlrfTB4,1209
 fleet/agent/gemini_cua/__init__.py,sha256=EqjnPPhWWUI3Gk5FrWKfajVYPn_nlO6tTl_jPdeHSXU,205
-fleet/agent/gemini_cua/agent.py,sha256=D-DpAa3DqDDGQiLc2wIiq6CC2ElmPL9wWhwMb8pWP-4,18263
-fleet/agent/gemini_cua/mcp_server.py,sha256=_MvYPDi8wE3w3AzMqG03ab0nskTI7QKTurzf2CdPiD8,7480
+fleet/agent/gemini_cua/agent.py,sha256=vm8s0B6yg3gyeMmRn9lI1pBOIChUBBPtO6vtZFQBgQk,19844
+fleet/agent/gemini_cua/mcp_server.py,sha256=_-LBmmnLp7krXrmGVHtX9ofFtRGUoeNhrLg5eshl3oI,7594
+fleet/agent/gemini_cua/playwright_utils.py,sha256=UZrlJJC5z3deQfvSAo03g-ajNkN52b333gBjqvQ6Qa0,15546
+fleet/agent/gemini_cua/requirements.txt,sha256=WJR0rfBCyDzeQg1ivLpKclVdpeFIlbaGZIErlGEr-IE,69
+fleet/agent/gemini_cua/start.sh,sha256=sw05agVHRcFSc1LRgBHWYMI62zu2HDFkH7bvrTUvw7c,827
 fleet/env/__init__.py,sha256=BVPZ4AYTznL6AYNrVmjr1yLF16qBcT1U56jWwF7AJ5o,964
 fleet/env/client.py,sha256=N9oF2hGSNmfVacTnyQhIEocoN3BA5fMa-arbpVeNi-E,3221
 fleet/eval/__init__.py,sha256=ng9V-KTsdW-bOOyF7UUGvABA9x0npeuYzMdS3MUd4nQ,389
@@ -88,7 +92,7 @@ fleet/verifiers/decorator.py,sha256=RuTjjDijbicNfMSjA7HcTpKueEki5dzNOdTuHS7UoZs,
 fleet/verifiers/parse.py,sha256=qz9AfJrTbjlg-LU-lE8Ciqi7Yt2a8-cs17FdpjTLhMk,8550
 fleet/verifiers/sql_differ.py,sha256=TqTLWyK3uOyLbitT6HYzYEzuSFC39wcyhgk3rcm__k8,6525
 fleet/verifiers/verifier.py,sha256=iqGevW7dSd0J5RdRQjpu-zioy_FYAXnzMfkuB3-QmO0,14601
-fleet_python-0.2.83.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fleet_python-0.2.85.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
 scripts/fix_sync_imports.py,sha256=X9fWLTpiPGkSHsjyQUDepOJkxOqw1DPj7nd8wFlFqLQ,8368
 scripts/unasync.py,sha256=vWVQxRWX8SRZO5cmzEhpvnG_REhCWXpidIGIpWmEcvI,696
 tests/__init__.py,sha256=Re1SdyxH8NfyL1kjhi7SQkGP1mYeWB-D6UALqdIMd8I,35
@@ -98,8 +102,8 @@ tests/test_instance_dispatch.py,sha256=CvU4C3LBIqsYZdEsEFfontGjyxAZfVYyXnGwxyIvX
 tests/test_sqlite_resource_dual_mode.py,sha256=Mh8jBd-xsIGDYFsOACKKK_5DXMUYlFFS7W-jaY6AjG4,8734
 tests/test_sqlite_shared_memory_behavior.py,sha256=fKx_1BmLS3b8x-9pMgjMycpnaHWY8P-2ZuXEspx6Sbw,4082
 tests/test_verifier_from_string.py,sha256=Lxi3TpFHFb-hG4-UhLKZJkqo84ax9YJY8G6beO-1erM,13581
-fleet_python-0.2.83.dist-info/METADATA,sha256=SOgVOcZ9dToWUp5_Huk5aSvWhnhlRjEHEDbRdJ5qpF0,4166
-fleet_python-0.2.83.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fleet_python-0.2.83.dist-info/entry_points.txt,sha256=qKIQ326cHR5WyCd16QnrW-1DpcT0YyxVRDb3IlTyzTA,39
-fleet_python-0.2.83.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
-fleet_python-0.2.83.dist-info/RECORD,,
+fleet_python-0.2.85.dist-info/METADATA,sha256=CuG2c-jrKzWdgrw6n0jL1XwGsryKpafXW9jNEpbFqXA,4166
+fleet_python-0.2.85.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fleet_python-0.2.85.dist-info/entry_points.txt,sha256=qKIQ326cHR5WyCd16QnrW-1DpcT0YyxVRDb3IlTyzTA,39
+fleet_python-0.2.85.dist-info/top_level.txt,sha256=qb1zIbtEktyhRFZdqVytwg54l64qtoZL0wjHB4bUg3c,29
+fleet_python-0.2.85.dist-info/RECORD,,