connectonion 0.6.1__py3-none-any.whl → 0.6.3__py3-none-any.whl

connectonion/__init__.py

@@ -1,13 +1,23 @@
-"""ConnectOnion - A simple agent framework with behavior tracking."""
+"""
+Purpose: Main package entry point exposing public API for ConnectOnion framework
+LLM-Note:
+  Dependencies: imports from [core/, logger.py, llm_do.py, transcribe.py, prompts.py, debug/, useful_tools/, network/, address.py] | imported by [user code, tests/, examples/] | no direct tests (integration tests import from here)
+  Data flow: loads .env from cwd via load_dotenv() → exports all public API symbols → user imports `from connectonion import Agent, llm_do, ...`
+  State/Effects: auto-loads .env file from current working directory (NOT module directory) at import time
+  Integration: exposes complete public API: Agent, LLM, Logger, create_tool_from_function, llm_do, transcribe, xray, event decorators, built-in tools, networking functions | __all__ defines explicit public exports
+  Performance: .env loading happens once at first import (dotenv caches)
+  Errors: none (import errors bubble from submodules)
+ConnectOnion - A simple agent framework with behavior tracking.
+"""
 
-__version__ = "0.6.1"
+__version__ = "0.6.3"
 
 # Auto-load .env files for the entire framework
 from dotenv import load_dotenv
 from pathlib import Path as _Path
 
-# Load from current working directory (where user runs their script)
-# NOT from the module's location (framework directory)
+# SDK only loads from current working directory (where user runs their script)
+# CLI commands (co ai, co browser, etc.) handle global fallback separately
 load_dotenv(_Path.cwd() / ".env")
 
 from .core import Agent, LLM, create_tool_from_function
@@ -27,12 +37,20 @@ from .llm_do import llm_do
 from .transcribe import transcribe
 from .prompts import load_system_prompt
 from .debug import xray, auto_debug_exception, replay, xray_replay
-from .useful_tools import send_email, get_emails, mark_read, mark_unread, Memory, Gmail, GoogleCalendar, Outlook, MicrosoftCalendar, WebFetch, Shell, DiffWriter, pick, yes_no, autocomplete, TodoList, SlashCommand
-from .network import connect, RemoteAgent, host, create_app, Connection
+from .useful_tools import (
+    send_email, get_emails, mark_read, mark_unread,
+    Memory, Gmail, GoogleCalendar, Outlook, MicrosoftCalendar,
+    WebFetch, Shell, bash, DiffWriter, MODE_NORMAL, MODE_AUTO, MODE_PLAN,
+    pick, yes_no, autocomplete, TodoList, SlashCommand,
+    # Claude Code-style file tools
+    read_file, edit, multi_edit, glob, grep, write, FileWriter,
+)
+from .network import connect, RemoteAgent, Response, host, create_app, IO
 from .network import relay, announce
 from . import address
 
 __all__ = [
+    # Core
     "Agent",
     "LLM",
     "Logger",
@@ -43,10 +61,13 @@ __all__ = [
     "xray",
     "replay",
     "xray_replay",
+    "auto_debug_exception",
+    # Email tools
     "send_email",
     "get_emails",
     "mark_read",
     "mark_unread",
+    # Class-based tools
     "Memory",
     "Gmail",
     "GoogleCalendar",
@@ -54,21 +75,37 @@ __all__ = [
     "MicrosoftCalendar",
     "WebFetch",
     "Shell",
+    "bash",
     "DiffWriter",
+    "MODE_NORMAL",
+    "MODE_AUTO",
+    "MODE_PLAN",
+    # TUI helpers
     "pick",
     "yes_no",
     "autocomplete",
+    # Task management
     "TodoList",
     "SlashCommand",
-    "auto_debug_exception",
+    # Claude Code-style file tools
+    "read_file",
+    "edit",
+    "multi_edit",
+    "glob",
+    "grep",
+    "write",
+    "FileWriter",
+    # Networking
     "connect",
     "RemoteAgent",
+    "Response",
     "host",
     "create_app",
-    "Connection",
+    "IO",
     "relay",
     "announce",
     "address",
+    # Event decorators
     "after_user_input",
     "before_llm",
     "after_llm",
@@ -77,5 +114,5 @@ __all__ = [
     "after_each_tool",
     "after_tools",
     "on_error",
-    "on_complete"
+    "on_complete",
 ]

connectonion/cli/__init__.py

@@ -1,3 +1,13 @@
-"""ConnectOnion CLI module."""
+"""
+Purpose: CLI module entry point with version metadata
+LLM-Note:
+  Dependencies: none | imported by [cli/main.py, setup.py] | no direct tests
+  Data flow: pure metadata (version string)
+  State/Effects: no state
+  Integration: exposes __version__ for CLI about/version commands
+  Performance: trivial
+  Errors: none
+ConnectOnion CLI module.
+"""
 
 __version__ = "0.0.1b5"

connectonion/cli/browser_agent/__init__.py

@@ -1,4 +1,14 @@
-"""Browser agent module for ConnectOnion CLI."""
+"""
+Purpose: Browser agent module exports for CLI browser automation
+LLM-Note:
+  Dependencies: imports from [browser.py] | imported by [cli/commands/browser_commands.py] | no direct tests
+  Data flow: re-exports execute_browser_command and BrowserAutomation
+  State/Effects: no state
+  Integration: exposes execute_browser_command(), BrowserAutomation class for `co browser` command
+  Performance: trivial
+  Errors: none
+Browser agent module for ConnectOnion CLI.
+"""
 
 from .browser import execute_browser_command, BrowserAutomation
 

connectonion/cli/browser_agent/browser.py

@@ -1,4 +1,13 @@
-"""Browser Agent for CLI - Natural language browser automation.
+"""
+Purpose: Natural language browser automation via Playwright with Chrome profile support
+LLM-Note:
+  Dependencies: imports from [playwright.sync_api, connectonion Agent/llm_do, cli/browser_agent/element_finder, pydantic, pathlib, dotenv] | imported by [cli/commands/browser_commands.py] | tested by [tests/cli/test_browser_agent.py]
+  Data flow: BrowserAutomation() initializes Playwright → copies Chrome profile to .browser_agent_profile/ → opens browser with context → provides tools (navigate, find_element, fill_form, screenshot, scroll, wait_for_login) → Agent uses these tools via natural language → element_finder.py uses vision LLM to locate elements | screenshots saved to .tmp/ directory
+  State/Effects: maintains browser/page/context state | copies Chrome profile on first run | writes screenshots to .tmp/{timestamp}.png | modifies form_data dict for form fills | auto-closes browser in __del__
+  Integration: exposes BrowserAutomation(use_chrome_profile, headless) with methods: navigate(url), find_element(description), fill_form(field_values), screenshot(viewport), scroll(direction, description), click(description), type_text(description, text), wait_for_login(seconds) | FormField Pydantic model for form parsing | used by `co browser` CLI command
+  Performance: headless by default (faster) | Chrome profile copy (one-time, slow first run) | vision model for element finding (slower but accurate) | screenshots base64-encoded for LLM analysis
+  Errors: returns error string if Playwright not installed | returns "Browser already open" if reinitializing | element not found returns descriptive error | Chrome must be closed when using profile
+Browser Agent for CLI - Natural language browser automation.
 
 This module provides a browser automation agent that understands natural language
 requests for browser operations via the ConnectOnion CLI.
@@ -20,6 +29,7 @@ from typing import Optional, List, Dict, Any
 from connectonion import Agent, llm_do
 from dotenv import load_dotenv
 from pydantic import BaseModel, Field
+from . import element_finder
 
 # Default screenshots directory
 SCREENSHOTS_DIR = Path.cwd() / ".tmp"
@@ -53,7 +63,7 @@ class BrowserAutomation:
     Supports Chrome profile for persistent sessions.
     """
 
-    def __init__(self, use_chrome_profile: bool = False, headless: bool = True):
+    def __init__(self, use_chrome_profile: bool = True, headless: bool = True):
         """Initialize browser automation.
 
         Args:
@@ -95,8 +105,9 @@ class BrowserAutomation:
         self.playwright = sync_playwright().start()
 
         if self.use_chrome_profile:
-            # Use Chromium with Chrome profile copy
-            chromium_profile = Path.cwd() / "chromium_automation_profile"
+            # Use Chromium with Chrome profile copy in global ~/.co/ folder
+            chromium_profile = Path.home() / ".co" / "browser_profile"
+            chromium_profile.parent.mkdir(parents=True, exist_ok=True)
 
             if not chromium_profile.exists():
                 import shutil
@@ -109,17 +120,28 @@ class BrowserAutomation:
                     source_profile = home / ".config/google-chrome"
 
                 if source_profile.exists():
+                    def safe_copy(src, dst):
+                        try:
+                            shutil.copy2(src, dst)
+                        except:
+                            pass  # Skip any file that can't be copied
+
                     shutil.copytree(
                         source_profile,
                         chromium_profile,
-                        ignore=shutil.ignore_patterns('*Cache*', '*cache*', 'Service Worker', 'ShaderCache'),
+                        ignore=shutil.ignore_patterns(
+                            '*Cache*', '*cache*', 'Service Worker', 'ShaderCache',
+                            'Singleton*', '*lock*', '*Lock*', '*.tmp', 'GPUCache',
+                            'Code Cache', 'DawnCache', 'GrShaderCache', 'blob_storage'
+                        ),
+                        copy_function=safe_copy,
                         dirs_exist_ok=True
                     )
 
             self.browser = self.playwright.chromium.launch_persistent_context(
                 str(chromium_profile),
                 headless=headless,
-                args=['--disable-blink-features=AutomationControlled'],
+                args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled'],
                 ignore_default_args=['--enable-automation'],
                 timeout=120000,
             )
@@ -127,10 +149,12 @@ class BrowserAutomation:
             self.page.add_init_script("""
                 Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
             """)
+            self.page.set_viewport_size({"width": 1920, "height": 1080})
             return f"Browser opened with Chrome profile: {chromium_profile}"
         else:
             self.browser = self.playwright.chromium.launch(headless=headless)
             self.page = self.browser.new_page()
+            self.page.set_viewport_size({"width": 1920, "height": 1080})
             return "Browser opened successfully"
 
     def go_to(self, url: str) -> str:
@@ -149,88 +173,94 @@ class BrowserAutomation:
     def find_element_by_description(self, description: str) -> str:
         """Find element using natural language description.
 
-        Uses AI to analyze HTML and find the best matching element.
+        Uses element_finder: LLM selects from indexed list, never generates CSS.
 
         Args:
             description: e.g., "the submit button", "email input field"
 
         Returns:
-            CSS selector for the element, or error message
+            Pre-built locator string, or error message
         """
         if not self.page:
             return "Browser not open"
 
-        html = self.page.content()
-
-        class ElementSelector(BaseModel):
-            selector: str = Field(..., description="CSS selector for the element")
-            confidence: float = Field(..., description="Confidence score 0-1")
-            explanation: str = Field(..., description="Why this element matches")
-
-        result = llm_do(
-            f"""Analyze this HTML and find the CSS selector for: "{description}"
-
-            HTML (first 15000 chars): {html[:15000]}
-
-            Return the most specific CSS selector that uniquely identifies this element.
-            """,
-            output=ElementSelector,
-            model="gpt-4o",
-            temperature=0.1
-        )
-
-        if self.page.locator(result.selector).count() > 0:
-            return result.selector
-        else:
-            return f"Found selector {result.selector} but element not on page"
+        element = element_finder.find_element(self.page, description)
+        if element:
+            return element.locator
+        return f"Could not find element matching: {description}"
 
     def click(self, description: str) -> str:
         """Click on an element using natural language description.
 
-        Args:
-            description: e.g., "the blue submit button", "link to contact page"
+        Uses element_finder: LLM selects from pre-built locators, never generates CSS.
         """
         if not self.page:
             return "Browser not open"
 
-        selector = self.find_element_by_description(description)
+        element = element_finder.find_element(self.page, description)
 
-        if selector.startswith("Could not") or selector.startswith("Found selector"):
-            if self.page.locator(f"text='{description}'").count() > 0:
-                self.page.click(f"text='{description}'")
-                return f"Clicked on '{description}' (by text)"
-            return selector
+        if not element:
+            # Fallback to simple text matching
+            text_locator = self.page.get_by_text(description)
+            if text_locator.count() > 0:
+                text_locator.first.click()
+                return f"Clicked on '{description}' (by text fallback)"
+            return f"Could not find element matching: {description}"
 
-        self.page.click(selector)
-        return f"Clicked on '{description}'"
+        # Try the locator with fresh bounding box
+        locator = self.page.locator(element.locator)
+
+        if locator.count() > 0:
+            box = locator.first.bounding_box()
+            if box:
+                x = box['x'] + box['width'] / 2
+                y = box['y'] + box['height'] / 2
+                self.page.mouse.click(x, y)
+                return f"Clicked [{element.index}] {element.tag} '{element.text}'"
+
+            locator.first.click(force=True)
+            return f"Clicked [{element.index}] {element.tag} '{element.text}' (force)"
+
+        # Fallback: use original coordinates
+        x = element.x + element.width // 2
+        y = element.y + element.height // 2
+        self.page.mouse.click(x, y)
+        return f"Clicked [{element.index}] '{element.text}' at ({x}, {y})"
 
     def type_text(self, field_description: str, text: str) -> str:
         """Type text into a form field.
 
-        Args:
-            field_description: e.g., "email field", "password input"
-            text: The text to type
+        Uses element_finder: LLM selects from pre-built locators, never generates CSS.
         """
         if not self.page:
             return "Browser not open"
 
-        selector = self.find_element_by_description(field_description)
-
-        if selector.startswith("Could not") or selector.startswith("Found selector"):
-            for fallback in [
-                f"input[placeholder*='{field_description}' i]",
-                f"[aria-label*='{field_description}' i]",
-                f"input[name*='{field_description}' i]"
-            ]:
-                if self.page.locator(fallback).count() > 0:
-                    self.page.fill(fallback, text)
-                    self.form_data[field_description] = text
-                    return f"Typed into {field_description}"
-            return f"Could not find field '{field_description}'"
-
-        self.page.fill(selector, text)
+        element = element_finder.find_element(self.page, field_description)
+
+        if not element:
+            # Fallback to placeholder matching
+            placeholder_locator = self.page.get_by_placeholder(field_description)
+            if placeholder_locator.count() > 0:
+                placeholder_locator.first.fill(text)
+                self.form_data[field_description] = text
+                return f"Typed into '{field_description}'"
+            return f"Could not find field: {field_description}"
+
+        # Try the pre-built locator
+        locator = self.page.locator(element.locator)
+
+        if locator.count() > 0:
+            locator.first.fill(text)
+            self.form_data[field_description] = text
+            return f"Typed into [{element.index}] {element.tag}"
+
+        # Fallback: click then type
+        x = element.x + element.width // 2
+        y = element.y + element.height // 2
+        self.page.mouse.click(x, y)
+        self.page.keyboard.type(text)
         self.form_data[field_description] = text
-        return f"Typed into {field_description}"
+        return f"Typed into [{element.index}] at ({x}, {y})"
 
     def get_text(self) -> str:
         """Get all visible text from the page."""
@@ -306,27 +336,6 @@ class BrowserAutomation:
         self.page.set_viewport_size({"width": width, "height": height})
         return f"Viewport set to {width}x{height}"
 
-    def screenshot_mobile(self, url: str = None) -> str:
-        """Take screenshot with iPhone viewport (390x844)."""
-        if url:
-            self.go_to(url)
-        self.set_viewport(390, 844)
-        return self.take_screenshot()
-
-    def screenshot_tablet(self, url: str = None) -> str:
-        """Take screenshot with iPad viewport (768x1024)."""
-        if url:
-            self.go_to(url)
-        self.set_viewport(768, 1024)
-        return self.take_screenshot()
-
-    def screenshot_desktop(self, url: str = None) -> str:
-        """Take screenshot with desktop viewport (1920x1080)."""
-        if url:
-            self.go_to(url)
-        self.set_viewport(1920, 1080)
-        return self.take_screenshot()
-
     def find_forms(self) -> List[FormField]:
         """Find all form fields on the current page."""
         if not self.page:
@@ -440,69 +449,13 @@ class BrowserAutomation:
         return f"Waited for {seconds} seconds"
 
     def scroll(self, times: int = 5, description: str = "the main content area") -> str:
-        """Universal scroll with automatic strategy selection.
-
-        Tries multiple strategies until one works:
-        1. AI-generated strategy (analyzes page structure)
-        2. Element scrolling
-        3. Page scrolling
+        """Universal scroll with AI strategy and fallback.
 
-        Args:
-            times: Number of scroll iterations
-            description: What to scroll (e.g., "the email list")
-
-        Returns:
-            Status message with successful strategy
+        Tries: AI-generated → Element scroll → Page scroll
+        Verifies success with screenshot comparison.
         """
-        from . import scroll_strategies
-        return scroll_strategies.scroll_with_verification(
-            page=self.page,
-            take_screenshot=self.take_screenshot,
-            times=times,
-            description=description
-        )
-
-    def scroll_page(self, direction: str = "down", amount: int = 1000) -> str:
-        """Scroll the page in a direction.
-
-        Args:
-            direction: "down", "up", "top", or "bottom"
-            amount: Pixels to scroll (ignored for "bottom"/"top")
-        """
-        if not self.page:
-            return "Browser not open"
-
-        if direction == "bottom":
-            self.page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
-            return "Scrolled to bottom of page"
-        elif direction == "top":
-            self.page.evaluate("window.scrollTo(0, 0)")
-            return "Scrolled to top of page"
-        elif direction == "down":
-            self.page.evaluate(f"window.scrollBy(0, {amount})")
-            return f"Scrolled down {amount} pixels"
-        elif direction == "up":
-            self.page.evaluate(f"window.scrollBy(0, -{amount})")
-            return f"Scrolled up {amount} pixels"
-        else:
-            return f"Unknown direction: {direction}"
-
-    def scroll_element(self, selector: str, amount: int = 1000) -> str:
-        """Scroll a specific element by CSS selector."""
-        if not self.page:
-            return "Browser not open"
-
-        result = self.page.evaluate(f"""
-            (() => {{
-                const element = document.querySelector('{selector}');
-                if (!element) return 'Element not found: {selector}';
-                const beforeScroll = element.scrollTop;
-                element.scrollTop += {amount};
-                const afterScroll = element.scrollTop;
-                return `Scrolled from ${{beforeScroll}}px to ${{afterScroll}}px`;
-            }})()
-        """)
-        return result
+        from . import scroll
+        return scroll.scroll(self.page, self.take_screenshot, times, description)
 
     def wait_for_manual_login(self, site_name: str = "the website") -> str:
         """Pause automation for user to login manually.

connectonion/cli/browser_agent/element_finder.py

@@ -0,0 +1,147 @@
+"""
+Purpose: Find interactive elements on web pages using natural language descriptions via vision LLM
+LLM-Note:
+  Dependencies: imports from [playwright.sync_api Page, connectonion llm_do, pydantic, pathlib] | imported by [cli/browser_agent/browser.py] | tested by [tests/cli/test_element_finder.py]
+  Data flow: extract_elements(page) → evaluates extract_elements.js → injects data-browser-agent-id on all interactive elements → returns list[InteractiveElement] with locators | find_element(page, description, elements) → formats elements for LLM → calls llm_do with vision model + screenshot → LLM selects matching element by index → returns InteractiveElement with pre-built locator
+  State/Effects: modifies DOM by injecting data-browser-agent-id attributes (temporary, removed on navigation) | takes screenshot for vision analysis | no persistent state
+  Integration: exposes extract_elements(page) → list[InteractiveElement], find_element(page, description, elements, screenshot) → InteractiveElement|None, highlight_element(page, element) for visual feedback | InteractiveElement model has tag, text, role, aria_label, placeholder, x, y, width, height, locator | ElementMatch model for LLM response
+  Performance: JavaScript extraction is fast | LLM matching uses vision model (slower) | limits to 150 elements for context window | pre-built locators (no retry needed)
+  Errors: returns None if no matching element found | raises if Playwright page not available | element may be stale if page navigates
+Element Finder - Find interactive elements by natural language description.
+
+Inspired by browser-use (https://github.com/browser-use/browser-use).
+
+Architecture:
+1. JavaScript injects `data-browser-agent-id` into each interactive element
+2. LLM SELECTS from indexed element list, never GENERATES CSS selectors
+3. Pre-built locators are guaranteed to work
+
+Usage:
+    elements = extract_elements(page)
+    element = find_element(page, "the login button", elements)
+    page.locator(element.locator).click()
+"""
+
+from typing import List, Optional
+from pathlib import Path
+from pydantic import BaseModel, Field
+from connectonion import llm_do
+
+
+# Load JavaScript and prompt from files
+_BASE_DIR = Path(__file__).parent
+_EXTRACT_JS = (_BASE_DIR / "scripts" / "extract_elements.js").read_text()
+_ELEMENT_MATCHER_PROMPT = (_BASE_DIR / "prompts" / "element_matcher.md").read_text()
+
+
+class InteractiveElement(BaseModel):
+    """An interactive element on the page with pre-built locator."""
+    index: int
+    tag: str
+    text: str = ""
+    role: Optional[str] = None
+    aria_label: Optional[str] = None
+    placeholder: Optional[str] = None
+    input_type: Optional[str] = None
+    href: Optional[str] = None
+    x: int = 0
+    y: int = 0
+    width: int = 0
+    height: int = 0
+    locator: str = ""
+
+
+class ElementMatch(BaseModel):
+    """LLM's element selection result."""
+    index: int = Field(..., description="Index of the matching element")
+    confidence: float = Field(..., description="Confidence 0-1")
+    reasoning: str = Field(..., description="Why this element matches")
+
+
+def extract_elements(page) -> List[InteractiveElement]:
+    """Extract all interactive elements from the page.
+
+    Returns elements with:
+    - Bounding boxes (for position matching with screenshot)
+    - Pre-built Playwright locators (guaranteed to work)
+    - Text/aria/placeholder for LLM matching
+    """
+    raw = page.evaluate(_EXTRACT_JS)
+    return [InteractiveElement(**el) for el in raw]
+
+
+def format_elements_for_llm(elements: List[InteractiveElement], max_count: int = 150) -> str:
+    """Format elements as compact list for LLM context.
+
+    Format: [index] tag "text" pos=(x,y) {extra info}
+    """
+    lines = []
+    for el in elements[:max_count]:
+        parts = [f"[{el.index}]", el.tag]
+
+        if el.text:
+            parts.append(f'"{el.text}"')
+        elif el.placeholder:
+            parts.append(f'placeholder="{el.placeholder}"')
+        elif el.aria_label:
+            parts.append(f'aria="{el.aria_label}"')
+
+        parts.append(f"pos=({el.x},{el.y})")
+
+        if el.input_type and el.tag == 'input':
+            parts.append(f"type={el.input_type}")
+
+        if el.role:
+            parts.append(f"role={el.role}")
+
+        if el.href:
+            href_short = el.href.split('?')[0][-30:]
+            parts.append(f"href=...{href_short}")
+
+        lines.append(' '.join(parts))
+
+    return '\n'.join(lines)
+
+
+def find_element(
+    page,
+    description: str,
+    elements: List[InteractiveElement] = None
+) -> Optional[InteractiveElement]:
+    """Find an interactive element by natural language description.
+
+    This is the core function. LLM SELECTS from pre-built options.
+
+    Args:
+        page: Playwright page
+        description: Natural language like "the login button" or "email field"
+        elements: Pre-extracted elements (will extract if not provided)
+
+    Returns:
+        Matching InteractiveElement with pre-built locator, or None
+    """
+    if elements is None:
+        elements = extract_elements(page)
+
+    if not elements:
+        return None
+
+    element_list = format_elements_for_llm(elements)
+
+    # Build prompt from template
+    prompt = _ELEMENT_MATCHER_PROMPT.format(
+        description=description,
+        element_list=element_list
+    )
+
+    result = llm_do(
+        prompt,
+        output=ElementMatch,
+        model="co/gemini-2.5-flash",
+        temperature=0.1
+    )
+
+    if 0 <= result.index < len(elements):
+        return elements[result.index]
+
+    return None