connectonion 0.6.1__py3-none-any.whl → 0.6.2__py3-none-any.whl

connectonion/__init__.py

@@ -1,6 +1,6 @@
 """ConnectOnion - A simple agent framework with behavior tracking."""
 
-__version__ = "0.6.1"
+__version__ = "0.6.2"
 
 # Auto-load .env files for the entire framework
 from dotenv import load_dotenv

connectonion/cli/browser_agent/browser.py

@@ -20,6 +20,7 @@ from typing import Optional, List, Dict, Any
 from connectonion import Agent, llm_do
 from dotenv import load_dotenv
 from pydantic import BaseModel, Field
+from . import element_finder
 
 # Default screenshots directory
 SCREENSHOTS_DIR = Path.cwd() / ".tmp"
@@ -53,7 +54,7 @@ class BrowserAutomation:
     Supports Chrome profile for persistent sessions.
     """
 
-    def __init__(self, use_chrome_profile: bool = False, headless: bool = True):
+    def __init__(self, use_chrome_profile: bool = True, headless: bool = True):
         """Initialize browser automation.
 
         Args:
@@ -109,17 +110,28 @@ class BrowserAutomation:
                     source_profile = home / ".config/google-chrome"
 
                 if source_profile.exists():
+                    def safe_copy(src, dst):
+                        try:
+                            shutil.copy2(src, dst)
+                        except:
+                            pass  # Skip any file that can't be copied
+
                     shutil.copytree(
                         source_profile,
                         chromium_profile,
-                        ignore=shutil.ignore_patterns('*Cache*', '*cache*', 'Service Worker', 'ShaderCache'),
+                        ignore=shutil.ignore_patterns(
+                            '*Cache*', '*cache*', 'Service Worker', 'ShaderCache',
+                            'Singleton*', '*lock*', '*Lock*', '*.tmp', 'GPUCache',
+                            'Code Cache', 'DawnCache', 'GrShaderCache', 'blob_storage'
+                        ),
+                        copy_function=safe_copy,
                         dirs_exist_ok=True
                     )
 
             self.browser = self.playwright.chromium.launch_persistent_context(
                 str(chromium_profile),
                 headless=headless,
-                args=['--disable-blink-features=AutomationControlled'],
+                args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled'],
                 ignore_default_args=['--enable-automation'],
                 timeout=120000,
             )
@@ -127,10 +139,12 @@ class BrowserAutomation:
             self.page.add_init_script("""
                 Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
             """)
+            self.page.set_viewport_size({"width": 1920, "height": 1080})
             return f"Browser opened with Chrome profile: {chromium_profile}"
         else:
             self.browser = self.playwright.chromium.launch(headless=headless)
             self.page = self.browser.new_page()
+            self.page.set_viewport_size({"width": 1920, "height": 1080})
             return "Browser opened successfully"
 
     def go_to(self, url: str) -> str:
@@ -149,88 +163,94 @@ class BrowserAutomation:
     def find_element_by_description(self, description: str) -> str:
         """Find element using natural language description.
 
-        Uses AI to analyze HTML and find the best matching element.
+        Uses element_finder: LLM selects from indexed list, never generates CSS.
 
         Args:
             description: e.g., "the submit button", "email input field"
 
         Returns:
-            CSS selector for the element, or error message
+            Pre-built locator string, or error message
         """
         if not self.page:
             return "Browser not open"
 
-        html = self.page.content()
-
-        class ElementSelector(BaseModel):
-            selector: str = Field(..., description="CSS selector for the element")
-            confidence: float = Field(..., description="Confidence score 0-1")
-            explanation: str = Field(..., description="Why this element matches")
-
-        result = llm_do(
-            f"""Analyze this HTML and find the CSS selector for: "{description}"
-
-            HTML (first 15000 chars): {html[:15000]}
-
-            Return the most specific CSS selector that uniquely identifies this element.
-            """,
-            output=ElementSelector,
-            model="gpt-4o",
-            temperature=0.1
-        )
-
-        if self.page.locator(result.selector).count() > 0:
-            return result.selector
-        else:
-            return f"Found selector {result.selector} but element not on page"
+        element = element_finder.find_element(self.page, description)
+        if element:
+            return element.locator
+        return f"Could not find element matching: {description}"
 
     def click(self, description: str) -> str:
         """Click on an element using natural language description.
 
-        Args:
-            description: e.g., "the blue submit button", "link to contact page"
+        Uses element_finder: LLM selects from pre-built locators, never generates CSS.
         """
         if not self.page:
             return "Browser not open"
 
-        selector = self.find_element_by_description(description)
+        element = element_finder.find_element(self.page, description)
 
-        if selector.startswith("Could not") or selector.startswith("Found selector"):
-            if self.page.locator(f"text='{description}'").count() > 0:
-                self.page.click(f"text='{description}'")
-                return f"Clicked on '{description}' (by text)"
-            return selector
+        if not element:
+            # Fallback to simple text matching
+            text_locator = self.page.get_by_text(description)
+            if text_locator.count() > 0:
+                text_locator.first.click()
+                return f"Clicked on '{description}' (by text fallback)"
+            return f"Could not find element matching: {description}"
+
+        # Try the locator with fresh bounding box
+        locator = self.page.locator(element.locator)
 
-        self.page.click(selector)
-        return f"Clicked on '{description}'"
+        if locator.count() > 0:
+            box = locator.first.bounding_box()
+            if box:
+                x = box['x'] + box['width'] / 2
+                y = box['y'] + box['height'] / 2
+                self.page.mouse.click(x, y)
+                return f"Clicked [{element.index}] {element.tag} '{element.text}'"
+
+            locator.first.click(force=True)
+            return f"Clicked [{element.index}] {element.tag} '{element.text}' (force)"
+
+        # Fallback: use original coordinates
+        x = element.x + element.width // 2
+        y = element.y + element.height // 2
+        self.page.mouse.click(x, y)
+        return f"Clicked [{element.index}] '{element.text}' at ({x}, {y})"
 
     def type_text(self, field_description: str, text: str) -> str:
         """Type text into a form field.
 
-        Args:
-            field_description: e.g., "email field", "password input"
-            text: The text to type
+        Uses element_finder: LLM selects from pre-built locators, never generates CSS.
         """
         if not self.page:
             return "Browser not open"
 
-        selector = self.find_element_by_description(field_description)
-
-        if selector.startswith("Could not") or selector.startswith("Found selector"):
-            for fallback in [
-                f"input[placeholder*='{field_description}' i]",
-                f"[aria-label*='{field_description}' i]",
-                f"input[name*='{field_description}' i]"
-            ]:
-                if self.page.locator(fallback).count() > 0:
-                    self.page.fill(fallback, text)
-                    self.form_data[field_description] = text
-                    return f"Typed into {field_description}"
-            return f"Could not find field '{field_description}'"
-
-        self.page.fill(selector, text)
+        element = element_finder.find_element(self.page, field_description)
+
+        if not element:
+            # Fallback to placeholder matching
+            placeholder_locator = self.page.get_by_placeholder(field_description)
+            if placeholder_locator.count() > 0:
+                placeholder_locator.first.fill(text)
+                self.form_data[field_description] = text
+                return f"Typed into '{field_description}'"
+            return f"Could not find field: {field_description}"
+
+        # Try the pre-built locator
+        locator = self.page.locator(element.locator)
+
+        if locator.count() > 0:
+            locator.first.fill(text)
+            self.form_data[field_description] = text
+            return f"Typed into [{element.index}] {element.tag}"
+
+        # Fallback: click then type
+        x = element.x + element.width // 2
+        y = element.y + element.height // 2
+        self.page.mouse.click(x, y)
+        self.page.keyboard.type(text)
         self.form_data[field_description] = text
-        return f"Typed into {field_description}"
+        return f"Typed into [{element.index}] at ({x}, {y})"
 
     def get_text(self) -> str:
         """Get all visible text from the page."""
@@ -306,27 +326,6 @@ class BrowserAutomation:
         self.page.set_viewport_size({"width": width, "height": height})
         return f"Viewport set to {width}x{height}"
 
-    def screenshot_mobile(self, url: str = None) -> str:
-        """Take screenshot with iPhone viewport (390x844)."""
-        if url:
-            self.go_to(url)
-        self.set_viewport(390, 844)
-        return self.take_screenshot()
-
-    def screenshot_tablet(self, url: str = None) -> str:
-        """Take screenshot with iPad viewport (768x1024)."""
-        if url:
-            self.go_to(url)
-        self.set_viewport(768, 1024)
-        return self.take_screenshot()
-
-    def screenshot_desktop(self, url: str = None) -> str:
-        """Take screenshot with desktop viewport (1920x1080)."""
-        if url:
-            self.go_to(url)
-        self.set_viewport(1920, 1080)
-        return self.take_screenshot()
-
     def find_forms(self) -> List[FormField]:
         """Find all form fields on the current page."""
         if not self.page:
@@ -440,69 +439,13 @@ class BrowserAutomation:
         return f"Waited for {seconds} seconds"
 
     def scroll(self, times: int = 5, description: str = "the main content area") -> str:
-        """Universal scroll with automatic strategy selection.
-
-        Tries multiple strategies until one works:
-        1. AI-generated strategy (analyzes page structure)
-        2. Element scrolling
-        3. Page scrolling
+        """Universal scroll with AI strategy and fallback.
 
-        Args:
-            times: Number of scroll iterations
-            description: What to scroll (e.g., "the email list")
-
-        Returns:
-            Status message with successful strategy
+        Tries: AI-generated → Element scroll → Page scroll
+        Verifies success with screenshot comparison.
         """
-        from . import scroll_strategies
-        return scroll_strategies.scroll_with_verification(
-            page=self.page,
-            take_screenshot=self.take_screenshot,
-            times=times,
-            description=description
-        )
-
-    def scroll_page(self, direction: str = "down", amount: int = 1000) -> str:
-        """Scroll the page in a direction.
-
-        Args:
-            direction: "down", "up", "top", or "bottom"
-            amount: Pixels to scroll (ignored for "bottom"/"top")
-        """
-        if not self.page:
-            return "Browser not open"
-
-        if direction == "bottom":
-            self.page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
-            return "Scrolled to bottom of page"
-        elif direction == "top":
-            self.page.evaluate("window.scrollTo(0, 0)")
-            return "Scrolled to top of page"
-        elif direction == "down":
-            self.page.evaluate(f"window.scrollBy(0, {amount})")
-            return f"Scrolled down {amount} pixels"
-        elif direction == "up":
-            self.page.evaluate(f"window.scrollBy(0, -{amount})")
-            return f"Scrolled up {amount} pixels"
-        else:
-            return f"Unknown direction: {direction}"
-
-    def scroll_element(self, selector: str, amount: int = 1000) -> str:
-        """Scroll a specific element by CSS selector."""
-        if not self.page:
-            return "Browser not open"
-
-        result = self.page.evaluate(f"""
-            (() => {{
-                const element = document.querySelector('{selector}');
-                if (!element) return 'Element not found: {selector}';
-                const beforeScroll = element.scrollTop;
-                element.scrollTop += {amount};
-                const afterScroll = element.scrollTop;
-                return `Scrolled from ${{beforeScroll}}px to ${{afterScroll}}px`;
-            }})()
-        """)
-        return result
+        from . import scroll
+        return scroll.scroll(self.page, self.take_screenshot, times, description)
 
     def wait_for_manual_login(self, site_name: str = "the website") -> str:
         """Pause automation for user to login manually.

connectonion/cli/browser_agent/element_finder.py

@@ -0,0 +1,139 @@
+"""
+Element Finder - Find interactive elements by natural language description.
+
+Inspired by browser-use (https://github.com/browser-use/browser-use).
+
+Architecture:
+1. JavaScript injects `data-browser-agent-id` into each interactive element
+2. LLM SELECTS from indexed element list, never GENERATES CSS selectors
+3. Pre-built locators are guaranteed to work
+
+Usage:
+    elements = extract_elements(page)
+    element = find_element(page, "the login button", elements)
+    page.locator(element.locator).click()
+"""
+
+from typing import List, Optional
+from pathlib import Path
+from pydantic import BaseModel, Field
+from connectonion import llm_do
+
+
+# Load JavaScript and prompt from files
+_BASE_DIR = Path(__file__).parent
+_EXTRACT_JS = (_BASE_DIR / "scripts" / "extract_elements.js").read_text()
+_ELEMENT_MATCHER_PROMPT = (_BASE_DIR / "prompts" / "element_matcher.md").read_text()
+
+
+class InteractiveElement(BaseModel):
+    """An interactive element on the page with pre-built locator."""
+    index: int
+    tag: str
+    text: str = ""
+    role: Optional[str] = None
+    aria_label: Optional[str] = None
+    placeholder: Optional[str] = None
+    input_type: Optional[str] = None
+    href: Optional[str] = None
+    x: int = 0
+    y: int = 0
+    width: int = 0
+    height: int = 0
+    locator: str = ""
+
+
+class ElementMatch(BaseModel):
+    """LLM's element selection result."""
+    index: int = Field(..., description="Index of the matching element")
+    confidence: float = Field(..., description="Confidence 0-1")
+    reasoning: str = Field(..., description="Why this element matches")
+
+
+def extract_elements(page) -> List[InteractiveElement]:
+    """Extract all interactive elements from the page.
+
+    Returns elements with:
+    - Bounding boxes (for position matching with screenshot)
+    - Pre-built Playwright locators (guaranteed to work)
+    - Text/aria/placeholder for LLM matching
+    """
+    raw = page.evaluate(_EXTRACT_JS)
+    return [InteractiveElement(**el) for el in raw]
+
+
+def format_elements_for_llm(elements: List[InteractiveElement], max_count: int = 150) -> str:
+    """Format elements as compact list for LLM context.
+
+    Format: [index] tag "text" pos=(x,y) {extra info}
+    """
+    lines = []
+    for el in elements[:max_count]:
+        parts = [f"[{el.index}]", el.tag]
+
+        if el.text:
+            parts.append(f'"{el.text}"')
+        elif el.placeholder:
+            parts.append(f'placeholder="{el.placeholder}"')
+        elif el.aria_label:
+            parts.append(f'aria="{el.aria_label}"')
+
+        parts.append(f"pos=({el.x},{el.y})")
+
+        if el.input_type and el.tag == 'input':
+            parts.append(f"type={el.input_type}")
+
+        if el.role:
+            parts.append(f"role={el.role}")
+
+        if el.href:
+            href_short = el.href.split('?')[0][-30:]
+            parts.append(f"href=...{href_short}")
+
+        lines.append(' '.join(parts))
+
+    return '\n'.join(lines)
+
+
+def find_element(
+    page,
+    description: str,
+    elements: List[InteractiveElement] = None
+) -> Optional[InteractiveElement]:
+    """Find an interactive element by natural language description.
+
+    This is the core function. LLM SELECTS from pre-built options.
+
+    Args:
+        page: Playwright page
+        description: Natural language like "the login button" or "email field"
+        elements: Pre-extracted elements (will extract if not provided)
+
+    Returns:
+        Matching InteractiveElement with pre-built locator, or None
+    """
+    if elements is None:
+        elements = extract_elements(page)
+
+    if not elements:
+        return None
+
+    element_list = format_elements_for_llm(elements)
+
+    # Build prompt from template
+    prompt = _ELEMENT_MATCHER_PROMPT.format(
+        description=description,
+        element_list=element_list
+    )
+
+    result = llm_do(
+        prompt,
+        output=ElementMatch,
+        model="co/gemini-2.5-flash",
+        temperature=0.1
+    )
+
+    if 0 <= result.index < len(elements):
+        return elements[result.index]
+
+    return None

connectonion/cli/browser_agent/highlight_screenshot.py

@@ -0,0 +1,174 @@
+"""
+Screenshot highlighting - draw bounding boxes and indices on screenshots.
+Inspired by browser-use's python_highlights.py approach.
+"""
+
+from PIL import Image, ImageDraw, ImageFont
+from pathlib import Path
+from typing import List
+import element_finder
+
+# Color scheme for different element types
+ELEMENT_COLORS = {
+    'button': '#FF6B6B',  # Red
+    'input': '#4ECDC4',   # Teal
+    'select': '#45B7D1',  # Blue
+    'a': '#96CEB4',       # Green
+    'textarea': '#FF8C42', # Orange
+    'div': '#DDA0DD',     # Light purple
+    'span': '#FFD93D',    # Yellow
+    'default': '#9B59B6', # Purple
+}
+
+
+def get_font(size: int = 14):
+    """Get a cross-platform font."""
+    font_paths = [
+        '/System/Library/Fonts/Arial.ttf',  # macOS
+        '/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf',  # Linux
+        'C:\\Windows\\Fonts\\arial.ttf',  # Windows
+    ]
+    for path in font_paths:
+        try:
+            return ImageFont.truetype(path, size)
+        except OSError:
+            continue
+    return ImageFont.load_default()
+
+
+def draw_dashed_rect(draw: ImageDraw.Draw, bbox: tuple, color: str, dash: int = 4, gap: int = 4):
+    """Draw a dashed rectangle."""
+    x1, y1, x2, y2 = bbox
+
+    def draw_dashed_line(start, end, is_horizontal: bool):
+        if is_horizontal:
+            x, y = start
+            while x < end[0]:
+                end_x = min(x + dash, end[0])
+                draw.line([(x, y), (end_x, y)], fill=color, width=2)
+                x += dash + gap
+        else:
+            x, y = start
+            while y < end[1]:
+                end_y = min(y + dash, end[1])
+                draw.line([(x, y), (x, end_y)], fill=color, width=2)
+                y += dash + gap
+
+    # Draw four sides
+    draw_dashed_line((x1, y1), (x2, y1), True)   # Top
+    draw_dashed_line((x2, y1), (x2, y2), False)  # Right
+    draw_dashed_line((x1, y2), (x2, y2), True)   # Bottom
+    draw_dashed_line((x1, y1), (x1, y2), False)  # Left
+
+
+def highlight_screenshot(
+    screenshot_path: str,
+    elements: List[element_finder.InteractiveElement],
+    output_path: str = None
+) -> str:
+    """Draw bounding boxes and indices on a screenshot.
+
+    Args:
+        screenshot_path: Path to the screenshot image
+        elements: List of InteractiveElement objects with bounding boxes
+        output_path: Optional output path (defaults to {original}_highlighted.png)
+
+    Returns:
+        Path to the highlighted screenshot
+    """
+    # Load image
+    image = Image.open(screenshot_path).convert('RGBA')
+    draw = ImageDraw.Draw(image)
+    font = get_font(14)
+    small_font = get_font(11)
+
+    for el in elements:
+        # Skip elements with no size
+        if el.width < 5 or el.height < 5:
+            continue
+
+        # Get color based on tag
+        color = ELEMENT_COLORS.get(el.tag, ELEMENT_COLORS['default'])
+
+        # Calculate bounding box
+        x1, y1 = el.x, el.y
+        x2, y2 = el.x + el.width, el.y + el.height
+
+        # Draw dashed bounding box
+        draw_dashed_rect(draw, (x1, y1, x2, y2), color)
+
+        # Draw index label
+        label = str(el.index)
+        bbox = draw.textbbox((0, 0), label, font=font)
+        label_w = bbox[2] - bbox[0]
+        label_h = bbox[3] - bbox[1]
+        padding = 3
+
+        # Position: top-center of element, or above if small
+        label_x = x1 + (el.width - label_w) // 2 - padding
+        if el.height < 40:
+            label_y = max(0, y1 - label_h - padding * 2 - 2)
+        else:
+            label_y = y1 + 2
+
+        # Draw label background
+        draw.rectangle(
+            [label_x, label_y,
+             label_x + label_w + padding * 2,
+             label_y + label_h + padding * 2],
+            fill=color,
+            outline='white',
+            width=1
+        )
+
+        # Draw label text
+        draw.text(
+            (label_x + padding, label_y + padding),
+            label,
+            fill='white',
+            font=font
+        )
+
+    # Save output
+    if not output_path:
+        p = Path(screenshot_path)
+        output_path = str(p.parent / f"{p.stem}_highlighted{p.suffix}")
+
+    image.save(output_path)
+    return output_path
+
+
+def highlight_current_page(page, output_path: str = "screenshots/highlighted.png") -> str:
+    """Take a screenshot and highlight all interactive elements.
+
+    Args:
+        page: Playwright page object
+        output_path: Path to save the highlighted screenshot
+
+    Returns:
+        Path to the highlighted screenshot
+    """
+    import os
+    from datetime import datetime
+
+    # Ensure directory exists
+    os.makedirs("screenshots", exist_ok=True)
+
+    # Take screenshot
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    screenshot_path = f"screenshots/raw_{timestamp}.png"
+    page.screenshot(path=screenshot_path)
+
+    # Extract elements
+    elements = element_finder.extract_elements(page)
+
+    # Generate output path
+    output_path = f"screenshots/highlighted_{timestamp}.png"
+
+    # Create highlighted version
+    result = highlight_screenshot(screenshot_path, elements, output_path)
+
+    # Clean up raw screenshot
+    os.remove(screenshot_path)
+
+    return result