code-puppy 0.0.358__tar.gz → 0.0.360__tar.gz

{code_puppy-0.0.358 → code_puppy-0.0.360}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-puppy
-Version: 0.0.358
+Version: 0.0.360
 Summary: Code generation agent
 Project-URL: repository, https://github.com/mpfaffenberger/code_puppy
 Project-URL: HomePage, https://github.com/mpfaffenberger/code_puppy

{code_puppy-0.0.358 → code_puppy-0.0.360}/code_puppy/agents/agent_qa_kitten.py

@@ -16,7 +16,7 @@ class QualityAssuranceKittenAgent(BaseAgent):
 
     @property
     def description(self) -> str:
-        return "Advanced web browser automation and quality assurance testing using Playwright with VQA capabilities"
+        return "Advanced web browser automation and quality assurance testing using Playwright with visual analysis capabilities"
 
     def get_available_tools(self) -> list[str]:
         """Get the list of tools available to Web Browser Puppy."""
@@ -63,8 +63,9 @@ class QualityAssuranceKittenAgent(BaseAgent):
             "browser_wait_for_element",
             "browser_highlight_element",
             "browser_clear_highlights",
-            # Screenshots and VQA (uses dedicated VQA agent for context management)
-            "browser_screenshot_vqa",
+            # Screenshots (returns BinaryContent for direct visual analysis)
+            "browser_screenshot_analyze",
+            "load_image_for_analysis",
             # Workflow management
             "browser_save_workflow",
             "browser_list_workflows",
@@ -78,7 +79,7 @@ You are Quality Assurance Kitten 🐱, an advanced autonomous browser automation
 
 You specialize in:
 🎯 **Quality Assurance Testing** - automated testing of web applications and user workflows
-👁️ **Visual verification** - taking screenshots and analyzing page content for bugs
+👁️ **Visual verification** - taking screenshots you can directly see and analyze for bugs
 🔍 **Element discovery** - finding elements using semantic locators and accessibility best practices
 📝 **Data extraction** - scraping content and gathering information from web pages
 🧪 **Web automation** - filling forms, clicking buttons, navigating sites with precision
@@ -117,12 +118,10 @@ For any browser task, follow this approach:
 
 ### Visual Verification Workflow
 - **Before critical actions**: Use browser_highlight_element to visually confirm
-- **After interactions**: Use browser_screenshot_vqa to verify results
-- **Ask specific questions**: The VQA tool requires a question like:
-  - "Is the login button visible?"
-  - "What error message is displayed?"
-  - "Is the form filled out correctly?"
-  - "What is the main heading text?"
+- **After interactions**: Use browser_screenshot_analyze to verify results
+- The screenshot is returned directly as an image you can see and analyze
+- No need to ask questions - just analyze what you see in the returned image
+- Use load_image_for_analysis to load mockups or reference images for comparison
 
 ### Form Input Best Practices
 - **ALWAYS check current values** with browser_get_value before typing
@@ -135,14 +134,15 @@ For any browser task, follow this approach:
 **When Element Discovery Fails:**
 1. Try different semantic locators first
 2. Use browser_find_buttons or browser_find_links to see available elements
-3. Take a screenshot with browser_screenshot_analyze to understand the page layout
+3. Take a screenshot with browser_screenshot_analyze to see and understand the page layout
 4. Only use XPath as absolute last resort
 
 **When Page Interactions Fail:**
 1. Check if element is visible with browser_wait_for_element
 2. Scroll element into view with browser_scroll_to_element
 3. Use browser_highlight_element to confirm element location
-4. Try browser_execute_js for complex interactions
+4. Take a screenshot with browser_screenshot_analyze to see the actual page state
+5. Try browser_execute_js for complex interactions
 
 ### JavaScript Execution
 - Use browser_execute_js for:
@@ -187,7 +187,7 @@ For any browser task, follow this approach:
 ## Specialized Capabilities
 
 🌐 **WCAG 2.2 Level AA Compliance**: Always prioritize accessibility in element discovery
-📸 **Visual Question Answering**: Use browser_screenshot_vqa for intelligent page analysis (uses dedicated VQA agent)
+📸 **Direct Visual Analysis**: Use browser_screenshot_analyze to see and analyze page content directly
 🚀 **Semantic Web Navigation**: Prefer role-based and label-based element discovery
 ⚡ **Playwright Power**: Full access to modern browser automation capabilities
 📋 **Workflow Management**: Save, load, and reuse automation patterns for consistency

{code_puppy-0.0.358 → code_puppy-0.0.360}/code_puppy/agents/event_stream_handler.py

@@ -119,6 +119,7 @@ async def event_stream_handler(
     tool_parts: set[int] = set()  # Track which parts are tool calls
     banner_printed: set[int] = set()  # Track if banner was already printed
     token_count: dict[int, int] = {}  # Track token count per text/tool part
+    tool_names: dict[int, str] = {}  # Track tool name per tool part index
     did_stream_anything = False  # Track if we streamed any content
 
     # Termflow streaming state for text parts
@@ -203,6 +204,8 @@ async def event_stream_handler(
                 streaming_parts.add(event.index)
                 tool_parts.add(event.index)
                 token_count[event.index] = 0  # Initialize token counter
+                # Capture tool name from the start event
+                tool_names[event.index] = part.tool_name or ""
                 # Track tool name for display
                 banner_printed.add(
                     event.index
@@ -253,20 +256,36 @@ async def event_stream_handler(
                             escaped = escape(delta.content_delta)
                             console.print(f"[dim]{escaped}[/dim]", end="")
                 elif isinstance(delta, ToolCallPartDelta):
-                    # For tool calls, count chunks received
-                    token_count[event.index] += 1
-                    # Get tool name if available
-                    tool_name = getattr(delta, "tool_name_delta", "")
+                    # For tool calls, estimate tokens from args_delta content
+                    # args_delta contains the streaming JSON arguments
+                    args_delta = getattr(delta, "args_delta", "") or ""
+                    if args_delta:
+                        # Rough estimate: 4 chars ≈ 1 token (same heuristic as subagent_stream_handler)
+                        estimated_tokens = max(1, len(args_delta) // 4)
+                        token_count[event.index] += estimated_tokens
+                    else:
+                        # Even empty deltas count as activity
+                        token_count[event.index] += 1
+
+                    # Update tool name if delta provides more of it
+                    tool_name_delta = getattr(delta, "tool_name_delta", "") or ""
+                    if tool_name_delta:
+                        tool_names[event.index] = (
+                            tool_names.get(event.index, "") + tool_name_delta
+                        )
+
+                    # Use stored tool name for display
+                    tool_name = tool_names.get(event.index, "")
                     count = token_count[event.index]
                     # Display with tool wrench icon and tool name
                     if tool_name:
                         console.print(
-                            f"  \U0001f527 Calling {tool_name}... {count} chunks   ",
+                            f"  \U0001f527 Calling {tool_name}... {count} token(s)   ",
                             end="\r",
                         )
                     else:
                         console.print(
-                            f"  \U0001f527 Calling tool... {count} chunks   ",
+                            f"  \U0001f527 Calling tool... {count} token(s)   ",
                             end="\r",
                         )
 
@@ -311,8 +330,9 @@ async def event_stream_handler(
                 elif event.index in banner_printed:
                     console.print()  # Final newline after streaming
 
-                # Clean up token count
+                # Clean up token count and tool names
                 token_count.pop(event.index, None)
+                tool_names.pop(event.index, None)
                 # Clean up all tracking sets
                 streaming_parts.discard(event.index)
                 thinking_parts.discard(event.index)

{code_puppy-0.0.358 → code_puppy-0.0.360}/code_puppy/config.py

@@ -98,7 +98,6 @@ _CURRENT_AUTOSAVE_ID: Optional[str] = None
 _model_validation_cache = {}
 _default_model_cache = None
 _default_vision_model_cache = None
-_default_vqa_model_cache = None
 
 
 def ensure_config_exists():
@@ -358,47 +357,6 @@ def _default_vision_model_from_models_json() -> str:
         return "gpt-4.1"
 
 
-def _default_vqa_model_from_models_json() -> str:
-    """Select a default VQA-capable model, preferring vision-ready options."""
-    global _default_vqa_model_cache
-
-    if _default_vqa_model_cache is not None:
-        return _default_vqa_model_cache
-
-    try:
-        from code_puppy.model_factory import ModelFactory
-
-        models_config = ModelFactory.load_config()
-        if models_config:
-            # Allow explicit VQA hints if present
-            for name, config in models_config.items():
-                if config.get("supports_vqa"):
-                    _default_vqa_model_cache = name
-                    return name
-
-            # Reuse multimodal heuristics before falling back to generic default
-            preferred_candidates = (
-                "gpt-4.1",
-                "gpt-4.1-mini",
-                "claude-4-0-sonnet",
-                "gemini-2.5-flash-preview-05-20",
-                "gpt-4.1-nano",
-            )
-            for candidate in preferred_candidates:
-                if candidate in models_config:
-                    _default_vqa_model_cache = candidate
-                    return candidate
-
-            _default_vqa_model_cache = _default_model_from_models_json()
-            return _default_vqa_model_cache
-
-        _default_vqa_model_cache = "gpt-4.1"
-        return "gpt-4.1"
-    except Exception:
-        _default_vqa_model_cache = "gpt-4.1"
-        return "gpt-4.1"
-
-
 def _validate_model_exists(model_name: str) -> bool:
     """Check if a model exists in models.json with caching to avoid redundant calls."""
     global _model_validation_cache
@@ -424,15 +382,10 @@ def _validate_model_exists(model_name: str) -> bool:
 
 def clear_model_cache():
     """Clear the model validation cache. Call this when models.json changes."""
-    global \
-        _model_validation_cache, \
-        _default_model_cache, \
-        _default_vision_model_cache, \
-        _default_vqa_model_cache
+    global _model_validation_cache, _default_model_cache, _default_vision_model_cache
     _model_validation_cache.clear()
     _default_model_cache = None
     _default_vision_model_cache = None
-    _default_vqa_model_cache = None
 
 
 def model_supports_setting(model_name: str, setting: str) -> bool:
@@ -503,21 +456,6 @@ def set_model_name(model: str):
     clear_model_cache()
 
 
-def get_vqa_model_name() -> str:
-    """Return the configured VQA model, falling back to the global model."""
-    stored_model = get_value("vqa_model_name")
-    if stored_model and _validate_model_exists(stored_model):
-        return stored_model
-    # Fall back to the global model if no specific VQA model is set
-    return get_global_model_name()
-
-
-def set_vqa_model_name(model: str):
-    """Persist the configured VQA model name and refresh caches."""
-    set_config_value("vqa_model_name", model or "")
-    clear_model_cache()
-
-
 def get_puppy_token():
     """Returns the puppy_token from config, or None if not set."""
     return get_value("puppy_token")

{code_puppy-0.0.358 → code_puppy-0.0.360}/code_puppy/tools/__init__.py

@@ -41,9 +41,6 @@ from code_puppy.tools.browser.browser_navigation import (
 from code_puppy.tools.browser.browser_screenshot import (
     register_take_screenshot_and_analyze,
 )
-from code_puppy.tools.browser.browser_screenshot_vqa import (
-    register_take_screenshot_and_analyze_vqa,
-)
 from code_puppy.tools.browser.browser_scripts import (
     register_browser_clear_highlights,
     register_browser_highlight_element,
@@ -146,9 +143,8 @@ TOOL_REGISTRY = {
     "browser_wait_for_element": register_wait_for_element,
     "browser_highlight_element": register_browser_highlight_element,
     "browser_clear_highlights": register_browser_clear_highlights,
-    # Browser Screenshots and VQA
+    # Browser Screenshots
     "browser_screenshot_analyze": register_take_screenshot_and_analyze,
-    "browser_screenshot_vqa": register_take_screenshot_and_analyze_vqa,
     # Browser Workflows
     "browser_save_workflow": register_save_workflow,
     "browser_list_workflows": register_list_workflows,

{code_puppy-0.0.358 → code_puppy-0.0.360}/code_puppy/tools/browser/__init__.py

@@ -11,7 +11,6 @@ from .camoufox_manager import (
     get_session_browser_manager,
     set_browser_session,
 )
-from .vqa_agent import VisualAnalysisResult, run_vqa_analysis, run_vqa_analysis_stream
 
 
 def format_terminal_banner(text: str) -> str:
@@ -35,7 +34,4 @@ __all__ = [
     "get_browser_session",
     "get_session_browser_manager",
     "set_browser_session",
-    "VisualAnalysisResult",
-    "run_vqa_analysis",
-    "run_vqa_analysis_stream",
 ]

{code_puppy-0.0.358 → code_puppy-0.0.360}/code_puppy/tools/browser/browser_screenshot.py

@@ -1,16 +1,16 @@
 """Screenshot tool for browser automation.
 
-Captures screenshots and returns them as base64 data that multimodal
-models can directly see and analyze - no separate VQA agent needed.
+Captures screenshots and returns them via ToolReturn with BinaryContent
+so multimodal models can directly see and analyze - no separate VQA agent needed.
 """
 
-import base64
+import time
 from datetime import datetime
 from pathlib import Path
 from tempfile import gettempdir, mkdtemp
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Union
 
-from pydantic_ai import RunContext
+from pydantic_ai import BinaryContent, RunContext, ToolReturn
 
 from code_puppy.messaging import emit_error, emit_info, emit_success
 from code_puppy.tools.common import generate_group_id
@@ -54,7 +54,6 @@ async def _capture_screenshot(
         result: Dict[str, Any] = {
             "success": True,
             "screenshot_bytes": screenshot_bytes,
-            "base64_data": base64.b64encode(screenshot_bytes).decode("utf-8"),
             "timestamp": timestamp,
         }
 
@@ -80,11 +79,11 @@ async def take_screenshot(
     full_page: bool = False,
     element_selector: Optional[str] = None,
     save_screenshot: bool = True,
-) -> Dict[str, Any]:
+) -> Union[ToolReturn, Dict[str, Any]]:
     """Take a screenshot of the browser page.
 
-    Returns the screenshot as base64-encoded PNG data that multimodal
-    models can directly see and analyze.
+    Returns a ToolReturn with BinaryContent so multimodal models can
+    directly see and analyze the screenshot.
 
     Args:
         full_page: Whether to capture full page or just viewport.
@@ -92,12 +91,11 @@ async def take_screenshot(
         save_screenshot: Whether to save the screenshot to disk.
 
     Returns:
-        Dict containing:
-            - success (bool): True if screenshot was captured.
-            - base64_image (str): Base64-encoded PNG image data.
-            - media_type (str): Always "image/png".
-            - screenshot_path (str): Path to saved file (if saved).
-            - error (str): Error message if unsuccessful.
+        ToolReturn containing:
+            - return_value: Success message with screenshot path
+            - content: List with description and BinaryContent image
+            - metadata: Screenshot details (path, target, timestamp)
+        Or Dict with error info if failed.
     """
     target = element_selector or ("full_page" if full_page else "viewport")
     group_id = generate_group_id("browser_screenshot", target)
@@ -122,15 +120,30 @@ async def take_screenshot(
 
         if not result["success"]:
             emit_error(result.get("error", "Screenshot failed"), message_group=group_id)
-            return result
-
-        return {
-            "success": True,
-            "base64_image": result["base64_data"],
-            "media_type": "image/png",
-            "screenshot_path": result.get("screenshot_path"),
-            "message": "Screenshot captured. The base64_image contains the browser view.",
-        }
+            return {"success": False, "error": result.get("error")}
+
+        screenshot_path = result.get("screenshot_path", "(not saved)")
+
+        # Return as ToolReturn with BinaryContent so the model can SEE the image!
+        return ToolReturn(
+            return_value=f"Screenshot captured successfully. Saved to: {screenshot_path}",
+            content=[
+                f"Here's the browser screenshot ({target}):",
+                BinaryContent(
+                    data=result["screenshot_bytes"],
+                    media_type="image/png",
+                ),
+                "Please analyze what you see and describe any relevant details.",
+            ],
+            metadata={
+                "success": True,
+                "screenshot_path": screenshot_path,
+                "target": target,
+                "full_page": full_page,
+                "element_selector": element_selector,
+                "timestamp": time.time(),
+            },
+        )
 
     except Exception as e:
         error_msg = f"Screenshot failed: {str(e)}"
@@ -146,19 +159,19 @@ def register_take_screenshot_and_analyze(agent):
         context: RunContext,
         full_page: bool = False,
         element_selector: Optional[str] = None,
-    ) -> Dict[str, Any]:
+    ) -> Union[ToolReturn, Dict[str, Any]]:
         """
         Take a screenshot of the browser page.
 
-        Returns the screenshot as base64 image data that you can see directly.
-        Use this to see what's displayed in the browser.
+        Returns the screenshot via ToolReturn with BinaryContent that you can
+        see directly. Use this to see what's displayed in the browser.
 
         Args:
             full_page: Capture full page (True) or just viewport (False).
             element_selector: Optional CSS selector to screenshot specific element.
 
         Returns:
-            Dict with base64_image (PNG data you can see), screenshot_path, etc.
+            ToolReturn with the screenshot image you can analyze, or error dict.
         """
         return await take_screenshot(
             full_page=full_page,