code-puppy 0.0.595__tar.gz → 0.0.597__tar.gz

{code_puppy-0.0.595 → code_puppy-0.0.597}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-puppy
-Version: 0.0.595
+Version: 0.0.597
 Summary: Code generation agent
 Project-URL: repository, https://github.com/mpfaffenberger/code_puppy
 Project-URL: HomePage, https://github.com/mpfaffenberger/code_puppy

{code_puppy-0.0.595 → code_puppy-0.0.597}/code_puppy/agents/agent_qa_kitten.py

@@ -35,6 +35,8 @@ class QualityAssuranceKittenAgent(BaseAgent):
             "browser_go_forward",
             "browser_reload",
             "browser_wait_for_load",
+            # Page state (DOM-first progression, PREFERRED for non-visual steps)
+            "browser_page_snapshot",
             # Element discovery (semantic locators preferred)
             "browser_find_by_role",
             "browser_find_by_text",
@@ -44,6 +46,10 @@ class QualityAssuranceKittenAgent(BaseAgent):
             "browser_find_buttons",
             "browser_find_links",
             "browser_xpath_query",  # Fallback when semantic locators fail
+            # Semantic interactions (accessibility-first, PREFERRED for progression)
+            "browser_click_by_role",
+            "browser_click_by_text",
+            "browser_set_text_by_label",
             # Element interactions
             "browser_click",
             "browser_double_click",
@@ -84,6 +90,36 @@ You specialize in:
 🧪 **Web automation** - filling forms, clicking buttons, navigating sites with precision
 🐛 **Bug detection** - identifying UI issues, broken functionality, and accessibility problems
 
+## DOM-First Progression vs Visual Validation (READ THIS FIRST)
+
+Every step you take is one of two kinds. Classify it before you act:
+
+**1. Functional / progression steps** (the default) - clicking through a
+flow, filling forms, navigating, checking that an action "worked" or that
+the page reached the expected state.
+- **PREFER DOM/text/accessibility locators and Playwright-style events.**
+- Use `browser_page_snapshot` to read page state cheaply, and the semantic
+  action tools (`browser_click_by_role`, `browser_click_by_text`,
+  `browser_set_text_by_label`) plus the `browser_find_by_*` locators.
+- **Validate success via the DOM**: URL, title, visible text, element
+  values, checked state, ARIA attributes - NOT screenshots.
+- **Do NOT take a screenshot just to decide whether an action progressed.**
+  Screenshots are fragile here: window moves, resizes, external monitors,
+  and harmless visual diffs cause false failures, and they're slow/expensive.
+
+**2. Visual / UX-UI validation steps** - rendering, layout, spacing,
+color, occlusion/overlap, responsive behavior, visual diffs, or comparison
+against a mockup/reference.
+- **THIS is when screenshots earn their keep.** Use
+  `browser_screenshot_analyze` and `load_image_for_analysis` freely.
+
+**Fallback rule:** If DOM-first strategies genuinely fail (element truly
+not locatable semantically), you may fall back to a screenshot to
+diagnose - but say so explicitly.
+
+**Always report which mode you used** ("DOM-first" or "visual fallback")
+when you describe a step or a failure, so problems are easy to diagnose.
+
 ## Core Workflow Philosophy
 
 For any browser task, follow this approach:
@@ -92,10 +128,10 @@ For any browser task, follow this approach:
 3. **Plan & Reason**: Break down complex tasks and explain your approach clearly
 4. **Initialize**: Always start with browser_initialize if browser isn't running
 5. **Navigate**: Use browser_navigate to reach the target page
-6. **Discover**: Use semantic locators (PREFERRED) for element discovery
-7. **Verify**: Use highlighting and screenshots to confirm elements
-8. **Act**: Interact with elements through clicks, typing, etc.
-9. **Validate**: Take screenshots or query DOM to verify actions worked
+6. **Discover**: Use `browser_page_snapshot` and semantic locators (PREFERRED) for element discovery
+7. **Verify**: Confirm the target via DOM state; reserve highlighting/screenshots for visual checks
+8. **Act**: Interact via semantic actions (click_by_role/text, set_text_by_label) or selector clicks/typing
+9. **Validate**: Query the DOM (snapshot/URL/text/value) to verify actions worked; screenshot only for visual assertions
 10. **Document Success**: Use browser_save_workflow to save successful patterns for future reuse
 
 ## Tool Usage Guidelines
@@ -108,17 +144,18 @@ For any browser task, follow this approach:
 ### Element Discovery Best Practices (ACCESSIBILITY FIRST! 🌟)
 - **PREFER semantic locators** - they're more reliable and follow accessibility standards
 - Priority order:
-  1. browser_find_by_role (button, link, textbox, heading, etc.)
-  2. browser_find_by_label (for form inputs)
-  3. browser_find_by_text (for visible text)
-  4. browser_find_by_placeholder (for input hints)
-  5. browser_find_by_test_id (for test-friendly elements)
-  6. browser_xpath_query (ONLY as last resort)
-
-### Visual Verification Workflow
-- **Before critical actions**: Use browser_highlight_element to visually confirm
-- **After interactions**: Use browser_screenshot_analyze to verify results
-- The screenshot is returned directly as an image you can see and analyze
+  1. browser_page_snapshot (cheap structured overview of the whole page)
+  2. browser_find_by_role (button, link, textbox, heading, etc.)
+  3. browser_find_by_label (for form inputs)
+  4. browser_find_by_text (for visible text)
+  5. browser_find_by_placeholder (for input hints)
+  6. browser_find_by_test_id (for test-friendly elements)
+  7. browser_xpath_query (ONLY as last resort)
+
+### Visual Verification Workflow (VISUAL ASSERTIONS ONLY)
+- Use screenshots for rendering, layout, color, occlusion, responsive, or mockup comparison - NOT to confirm functional progression
+- **Before critical visual checks**: Use browser_highlight_element to visually confirm
+- **For visual results**: Use browser_screenshot_analyze - returned directly as an image you can see and analyze
 - No need to ask questions - just analyze what you see in the returned image
 - Use load_image_for_analysis to load mockups or reference images for comparison
 
@@ -132,16 +169,18 @@ For any browser task, follow this approach:
 
 **When Element Discovery Fails:**
 1. Try different semantic locators first
-2. Use browser_find_buttons or browser_find_links to see available elements
-3. Take a screenshot with browser_screenshot_analyze to see and understand the page layout
-4. Only use XPath as absolute last resort
+2. Call browser_page_snapshot to see all buttons/links/inputs/text at once
+3. Use browser_find_buttons or browser_find_links to see available elements
+4. Only take a screenshot (browser_screenshot_analyze) as a visual fallback - note that you fell back to visual
+5. Only use XPath as absolute last resort
 
 **When Page Interactions Fail:**
 1. Check if element is visible with browser_wait_for_element
 2. Scroll element into view with browser_scroll_to_element
-3. Use browser_highlight_element to confirm element location
-4. Take a screenshot with browser_screenshot_analyze to see the actual page state
-5. Try browser_execute_js for complex interactions
+3. Re-check state with browser_page_snapshot (did the DOM change?)
+4. Use browser_highlight_element to confirm element location
+5. Take a screenshot with browser_screenshot_analyze only as a visual fallback - note that you fell back to visual
+6. Try browser_execute_js for complex interactions
 
 ### JavaScript Execution
 - Use browser_execute_js for:
@@ -197,7 +236,9 @@ For any browser task, follow this approach:
 - **ALWAYS use browser_initialize before any browser operations**
 - **ALWAYS close the browser at the end of every task** using browser_close
 - **PREFER semantic locators over XPath** - they're more maintainable and accessible
-- **Use visual verification for critical actions** - highlight elements and take screenshots
+- **PREFER DOM-first progression over screenshots** - use browser_page_snapshot and DOM state to validate functional steps; reserve screenshots for visual assertions
+- **Report your mode** - state whether a step used DOM-first or a visual fallback
+- **Use visual verification for critical VISUAL actions** - highlight elements and take screenshots for layout/color/rendering checks
 - **Be explicit about your reasoning** for complex workflows
 - **Handle errors gracefully** - provide helpful debugging information
 - **Follow accessibility best practices** - your automation should work for everyone

{code_puppy-0.0.595 → code_puppy-0.0.597}/code_puppy/tools/__init__.py

@@ -36,6 +36,14 @@ from code_puppy.tools.browser.browser_locators import (
     register_find_links,
     register_run_xpath_query,
 )
+from code_puppy.tools.browser.browser_page_snapshot import (
+    register_get_page_snapshot,
+)
+from code_puppy.tools.browser.browser_semantic_interactions import (
+    register_click_by_role,
+    register_click_by_text,
+    register_set_text_by_label,
+)
 from code_puppy.tools.browser.browser_navigation import (
     register_browser_go_back,
     register_browser_go_forward,
@@ -132,6 +140,11 @@ TOOL_REGISTRY = {
     "browser_xpath_query": register_run_xpath_query,
     "browser_find_buttons": register_find_buttons,
     "browser_find_links": register_find_links,
+    # Browser Semantic Interactions (accessibility-first, DOM progression)
+    "browser_page_snapshot": register_get_page_snapshot,
+    "browser_click_by_role": register_click_by_role,
+    "browser_click_by_text": register_click_by_text,
+    "browser_set_text_by_label": register_set_text_by_label,
     # Browser Element Interactions
     "browser_click": register_click_element,
     "browser_double_click": register_double_click_element,

code_puppy-0.0.597/code_puppy/tools/browser/browser_locator_resolver.py

@@ -0,0 +1,70 @@
+"""Shared Playwright semantic-locator resolution.
+
+Centralizes role/text/label/placeholder/test-id -> Playwright locator
+construction so the semantic discovery *and* action tools don't each
+reinvent it (DRY). Non-visual QA progression should lean on these
+accessibility-first locators instead of raw CSS/XPath or screenshots.
+"""
+
+from typing import Optional
+
+# Supported semantic strategies. The value documents which Playwright
+# ``page.get_by_*`` accessor backs each strategy.
+SEMANTIC_STRATEGIES: dict[str, str] = {
+    "role": "get_by_role",
+    "text": "get_by_text",
+    "label": "get_by_label",
+    "placeholder": "get_by_placeholder",
+    "test_id": "get_by_test_id",
+}
+
+
+def describe_target(strategy: str, value: str, name: Optional[str] = None) -> str:
+    """Human-readable description of a semantic target for diagnostics.
+
+    Keeps error messages deterministic and consistent across every tool
+    that resolves a locator.
+    """
+    if strategy == "role" and name:
+        return f"role={value!r} name={name!r}"
+    return f"{strategy}={value!r}"
+
+
+def resolve_locator(
+    page,
+    strategy: str,
+    value: str,
+    name: Optional[str] = None,
+    exact: bool = False,
+):
+    """Build a Playwright locator for a semantic strategy.
+
+    Args:
+        page: The active Playwright page.
+        strategy: One of :data:`SEMANTIC_STRATEGIES` keys.
+        value: The primary value (role name, text, label, placeholder, test id).
+        name: Accessible name, only used with the ``role`` strategy.
+        exact: Whether to match ``value``/``name`` exactly.
+
+    Returns:
+        A Playwright locator.
+
+    Raises:
+        ValueError: If ``strategy`` is not supported.
+    """
+    if strategy not in SEMANTIC_STRATEGIES:
+        supported = ", ".join(sorted(SEMANTIC_STRATEGIES))
+        raise ValueError(
+            f"Unknown locator strategy {strategy!r}. Expected one of: {supported}"
+        )
+
+    if strategy == "role":
+        return page.get_by_role(value, name=name, exact=exact)
+    if strategy == "text":
+        return page.get_by_text(value, exact=exact)
+    if strategy == "label":
+        return page.get_by_label(value, exact=exact)
+    if strategy == "placeholder":
+        return page.get_by_placeholder(value, exact=exact)
+    # test_id has no exact/name knobs in Playwright's accessor.
+    return page.get_by_test_id(value)

code_puppy-0.0.597/code_puppy/tools/browser/browser_page_snapshot.py

@@ -0,0 +1,148 @@
+"""Cheap structured page-state snapshot for DOM-first QA progression.
+
+Instead of taking a screenshot and visually reasoning about "did the
+action work", qa-kitten can call ``browser_page_snapshot`` to get a
+compact, deterministic view of the page: URL, title, headings, buttons
+(with accessible names), links, inputs, and key ARIA metadata. This is
+faster, cheaper, and immune to window moves / monitor differences.
+
+Screenshots remain the right tool for *visual* assertions (layout,
+color, occlusion, visual diff) - this is for functional progression.
+"""
+
+from typing import Any, Dict
+
+from pydantic_ai import RunContext
+
+from code_puppy.messaging import emit_info, emit_success
+from code_puppy.tools.common import generate_group_id
+
+from .browser_manager import get_session_browser_manager
+
+# One JS pass gathers everything so we make a single round-trip to the
+# page rather than N Playwright calls. ``limit`` caps each collection so
+# huge pages don't blow up the tool output.
+_SNAPSHOT_JS = """
+(limit) => {
+  const isVisible = (el) => {
+    const style = window.getComputedStyle(el);
+    if (style.visibility === 'hidden' || style.display === 'none') return false;
+    const rect = el.getBoundingClientRect();
+    return rect.width > 0 && rect.height > 0;
+  };
+  const accName = (el) =>
+    (el.getAttribute('aria-label')
+      || el.getAttribute('title')
+      || (el.textContent || '').trim()
+      || el.getAttribute('value')
+      || '').slice(0, 120);
+  const take = (nodes) => Array.from(nodes).filter(isVisible).slice(0, limit);
+
+  const headings = take(document.querySelectorAll('h1,h2,h3,h4,h5,h6')).map((el) => ({
+    level: el.tagName.toLowerCase(),
+    text: (el.textContent || '').trim().slice(0, 120),
+  }));
+  const buttons = take(
+    document.querySelectorAll('button,[role="button"],input[type="submit"],input[type="button"]')
+  ).map((el) => ({
+    name: accName(el),
+    disabled: el.disabled === true || el.getAttribute('aria-disabled') === 'true',
+  }));
+  const links = take(document.querySelectorAll('a[href]')).map((el) => ({
+    text: (el.textContent || '').trim().slice(0, 120),
+    href: el.getAttribute('href'),
+  }));
+  const inputs = take(
+    document.querySelectorAll('input,textarea,select')
+  ).map((el) => ({
+    tag: el.tagName.toLowerCase(),
+    type: el.getAttribute('type'),
+    name: el.getAttribute('name'),
+    placeholder: el.getAttribute('placeholder'),
+    label: el.getAttribute('aria-label'),
+    test_id: el.getAttribute('data-testid') || el.getAttribute('data-test-id'),
+    value: (el.value || '').slice(0, 120),
+    checked: el.checked === true,
+  }));
+  const landmarks = take(
+    document.querySelectorAll('[role],nav,main,header,footer,aside,form')
+  ).map((el) => ({
+    role: el.getAttribute('role') || el.tagName.toLowerCase(),
+    label: el.getAttribute('aria-label'),
+  }));
+
+  const bodyText = (document.body ? document.body.innerText : '') || '';
+  return {
+    url: window.location.href,
+    title: document.title,
+    visible_text: bodyText.replace(/\\s+/g, ' ').trim().slice(0, 2000),
+    headings,
+    buttons,
+    links,
+    inputs,
+    landmarks,
+  };
+}
+"""
+
+
+async def get_page_snapshot(limit: int = 25) -> Dict[str, Any]:
+    """Return a compact structured snapshot of the current page state.
+
+    Args:
+        limit: Max items collected per category (buttons, links, inputs...).
+
+    Returns:
+        Dict with ``success`` plus URL/title/visible_text and structured
+        lists of headings, buttons, links, inputs, and landmarks, or an
+        error dict if no page is available.
+    """
+    group_id = generate_group_id("browser_page_snapshot", "snapshot")
+    emit_info("BROWSER PAGE SNAPSHOT  gathering DOM state", message_group=group_id)
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        snapshot = await page.evaluate(_SNAPSHOT_JS, limit)
+
+        emit_success(
+            "Snapshot: "
+            f"{len(snapshot.get('buttons', []))} buttons, "
+            f"{len(snapshot.get('links', []))} links, "
+            f"{len(snapshot.get('inputs', []))} inputs",
+            message_group=group_id,
+        )
+
+        return {"success": True, **snapshot}
+
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+
+def register_get_page_snapshot(agent):
+    """Register the page snapshot tool."""
+
+    @agent.tool
+    async def browser_page_snapshot(
+        context: RunContext,
+        limit: int = 25,
+    ) -> Dict[str, Any]:
+        """
+        Get a cheap, structured snapshot of the current page's DOM state.
+
+        PREFER THIS over a screenshot when validating functional progression
+        (did the page change / did the element appear / what's the value now).
+        Returns URL, title, visible text excerpt, headings, buttons with
+        accessible names, links, inputs (with values/placeholders/test-ids),
+        and ARIA landmarks. Reserve screenshots for true visual assertions.
+
+        Args:
+            limit: Max items per category (buttons/links/inputs/etc).
+
+        Returns:
+            Dict with structured page state, or an error dict.
+        """
+        return await get_page_snapshot(limit=limit)

code_puppy-0.0.597/code_puppy/tools/browser/browser_semantic_interactions.py

@@ -0,0 +1,189 @@
+"""Semantic (accessibility-locator) interaction helpers.
+
+These let qa-kitten act *directly* through Playwright semantic locators
+(role/text/label) instead of first discovering an element and then
+translating it into a raw CSS/XPath selector. This is the preferred path
+for non-visual workflow progression: fewer round-trips, no fragile
+selectors, deterministic errors when nothing matches.
+"""
+
+from typing import Any, Dict, Optional
+
+from pydantic_ai import RunContext
+
+from code_puppy.messaging import emit_error, emit_info, emit_success
+from code_puppy.tools.common import generate_group_id
+
+from .browser_locator_resolver import describe_target, resolve_locator
+from .browser_manager import get_session_browser_manager
+
+
+async def _act_on_semantic(
+    strategy: str,
+    value: str,
+    action: str,
+    name: Optional[str] = None,
+    exact: bool = False,
+    text: Optional[str] = None,
+    timeout: int = 10000,
+) -> Dict[str, Any]:
+    """Resolve a semantic locator and perform an action on the first match.
+
+    Shared core for every semantic interaction so click/fill/check don't
+    duplicate the resolve-wait-act-diagnose dance (DRY).
+    """
+    target = describe_target(strategy, value, name)
+    group_id = generate_group_id(f"browser_{action}_by_{strategy}", target)
+    emit_info(
+        f"BROWSER {action.upper()} BY {strategy.upper()}  {target}",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        try:
+            locator = resolve_locator(page, strategy, value, name=name, exact=exact)
+        except ValueError as ve:
+            return {"success": False, "error": str(ve)}
+
+        element = locator.first
+
+        # Deterministic "no match" error instead of a raw Playwright timeout.
+        if await locator.count() == 0:
+            msg = f"No element matched {target}"
+            emit_error(msg, message_group=group_id)
+            return {"success": False, "error": msg, "target": target}
+
+        await element.wait_for(state="visible", timeout=timeout)
+
+        if action == "click":
+            await element.click(timeout=timeout)
+        elif action == "fill":
+            await element.fill(text or "", timeout=timeout)
+        elif action == "check":
+            await element.check(timeout=timeout)
+        else:  # pragma: no cover - guarded by callers
+            return {"success": False, "error": f"Unknown action {action!r}"}
+
+        emit_success(f"{action} on {target}", message_group=group_id)
+        result: Dict[str, Any] = {
+            "success": True,
+            "action": action,
+            "strategy": strategy,
+            "target": target,
+        }
+        if text is not None:
+            result["text"] = text
+        return result
+
+    except Exception as e:
+        emit_error(f"{action} failed: {str(e)}", message_group=group_id)
+        return {"success": False, "error": str(e), "target": target}
+
+
+async def click_by_role(
+    role: str, name: Optional[str] = None, exact: bool = False, timeout: int = 10000
+) -> Dict[str, Any]:
+    """Click the first element matching an ARIA role (and optional name)."""
+    return await _act_on_semantic(
+        "role", role, "click", name=name, exact=exact, timeout=timeout
+    )
+
+
+async def click_by_text(
+    text: str, exact: bool = False, timeout: int = 10000
+) -> Dict[str, Any]:
+    """Click the first element containing the given visible text."""
+    return await _act_on_semantic("text", text, "click", exact=exact, timeout=timeout)
+
+
+async def set_text_by_label(
+    label: str, text: str, exact: bool = False, timeout: int = 10000
+) -> Dict[str, Any]:
+    """Fill the input associated with the given label text."""
+    return await _act_on_semantic(
+        "label", label, "fill", exact=exact, text=text, timeout=timeout
+    )
+
+
+def register_click_by_role(agent):
+    """Register the click-by-role semantic tool."""
+
+    @agent.tool
+    async def browser_click_by_role(
+        context: RunContext,
+        role: str,
+        name: Optional[str] = None,
+        exact: bool = False,
+        timeout: int = 10000,
+    ) -> Dict[str, Any]:
+        """
+        Click an element by ARIA role (accessibility-first, PREFERRED for
+        non-visual progression). No need to discover a selector first.
+
+        Args:
+            role: ARIA role (button, link, tab, menuitem, etc.).
+            name: Optional accessible name to disambiguate.
+            exact: Match name exactly.
+            timeout: Timeout in milliseconds.
+
+        Returns:
+            Dict with action result or a deterministic no-match error.
+        """
+        return await click_by_role(role, name=name, exact=exact, timeout=timeout)
+
+
+def register_click_by_text(agent):
+    """Register the click-by-text semantic tool."""
+
+    @agent.tool
+    async def browser_click_by_text(
+        context: RunContext,
+        text: str,
+        exact: bool = False,
+        timeout: int = 10000,
+    ) -> Dict[str, Any]:
+        """
+        Click an element by its visible text (PREFERRED for non-visual
+        progression over screenshot-guided clicking).
+
+        Args:
+            text: Visible text to match.
+            exact: Match text exactly.
+            timeout: Timeout in milliseconds.
+
+        Returns:
+            Dict with action result or a deterministic no-match error.
+        """
+        return await click_by_text(text, exact=exact, timeout=timeout)
+
+
+def register_set_text_by_label(agent):
+    """Register the fill-by-label semantic tool."""
+
+    @agent.tool
+    async def browser_set_text_by_label(
+        context: RunContext,
+        label: str,
+        text: str,
+        exact: bool = False,
+        timeout: int = 10000,
+    ) -> Dict[str, Any]:
+        """
+        Type text into the input associated with a label (accessibility-first,
+        PREFERRED for non-visual form progression).
+
+        Args:
+            label: Label text of the target input.
+            text: Text to enter.
+            exact: Match label exactly.
+            timeout: Timeout in milliseconds.
+
+        Returns:
+            Dict with action result or a deterministic no-match error.
+        """
+        return await set_text_by_label(label, text, exact=exact, timeout=timeout)

{code_puppy-0.0.595 → code_puppy-0.0.597}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "code-puppy"
-version = "0.0.595"
+version = "0.0.597"
 description = "Code generation agent"
 readme = "README.md"
 requires-python = ">=3.11,<3.15"