connectonion 0.6.1__py3-none-any.whl → 0.6.3__py3-none-any.whl

connectonion/cli/browser_agent/scripts/extract_elements.js

@@ -0,0 +1,126 @@
+/**
+ * Extract interactive elements from the page with injected IDs.
+ *
+ * Inspired by browser-use (https://github.com/browser-use/browser-use).
+ *
+ * This script:
+ * 1. Finds all interactive elements (buttons, links, inputs, etc.)
+ * 2. Injects a unique `data-browser-agent-id` attribute into each
+ * 3. Returns element data with bounding boxes for LLM matching
+ */
+(() => {
+    const results = [];
+    let index = 0;
+
+    // Interactive element types
+    const INTERACTIVE_TAGS = new Set([
+        'a', 'button', 'input', 'select', 'textarea', 'label',
+        'details', 'summary', 'dialog'
+    ]);
+
+    const INTERACTIVE_ROLES = new Set([
+        'button', 'link', 'menuitem', 'menuitemcheckbox', 'menuitemradio',
+        'option', 'radio', 'switch', 'tab', 'checkbox', 'textbox',
+        'searchbox', 'combobox', 'listbox', 'slider', 'spinbutton'
+    ]);
+
+    // Check if element is visible
+    function isVisible(el) {
+        const style = window.getComputedStyle(el);
+        if (style.display === 'none') return false;
+        if (style.visibility === 'hidden') return false;
+        if (parseFloat(style.opacity) === 0) return false;
+
+        // Skip visually-hidden accessibility elements
+        const className = el.className || '';
+        if (typeof className === 'string' &&
+            (className.includes('visually-hidden') ||
+             className.includes('sr-only') ||
+             className.includes('screen-reader'))) {
+            return false;
+        }
+
+        const rect = el.getBoundingClientRect();
+        if (rect.width === 0 || rect.height === 0) return false;
+
+        // Skip elements that are clipped/hidden with CSS tricks
+        if (rect.width < 2 || rect.height < 2) return false;
+
+        // Check if in viewport (with some margin)
+        const margin = 100;
+        if (rect.bottom < -margin) return false;
+        if (rect.top > window.innerHeight + margin) return false;
+        if (rect.right < -margin) return false;
+        if (rect.left > window.innerWidth + margin) return false;
+
+        return true;
+    }
+
+    // Get clean text content
+    function getText(el) {
+        // For inputs, get value or placeholder
+        if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
+            return el.value || el.placeholder || '';
+        }
+        // For other elements, get inner text
+        const text = el.innerText || el.textContent || '';
+        return text.trim().replace(/\s+/g, ' ').substring(0, 80);
+    }
+
+    // Process all elements
+    document.querySelectorAll('*').forEach(el => {
+        const tag = el.tagName.toLowerCase();
+        const role = el.getAttribute('role');
+
+        // Check if interactive
+        const isInteractiveTag = INTERACTIVE_TAGS.has(tag);
+        const isInteractiveRole = role && INTERACTIVE_ROLES.has(role);
+        const isClickable = window.getComputedStyle(el).cursor === 'pointer';
+        const hasTabIndex = el.hasAttribute('tabindex') && el.tabIndex >= 0;
+        const hasClickHandler = el.onclick !== null || el.hasAttribute('onclick');
+
+        if (!isInteractiveTag && !isInteractiveRole && !isClickable &&
+            !hasTabIndex && !hasClickHandler) {
+            return;
+        }
+
+        // Skip hidden inputs
+        if (tag === 'input' && el.type === 'hidden') return;
+
+        // Skip empty elements with no text or useful attributes
+        const text = getText(el);
+        const ariaLabel = el.getAttribute('aria-label');
+        const placeholder = el.placeholder;
+        if (!text && !ariaLabel && !placeholder && tag !== 'input') return;
+
+        // Skip very small elements (likely icons)
+        const rect = el.getBoundingClientRect();
+        if (rect.width < 20 && rect.height < 20 && !text) return;
+
+        // Check visibility
+        if (!isVisible(el)) return;
+
+        // INJECT a unique ID attribute for reliable location
+        const highlightId = String(index);
+        el.setAttribute('data-browser-agent-id', highlightId);
+
+        results.push({
+            index: index++,
+            tag: tag,
+            text: text,
+            role: role,
+            aria_label: el.getAttribute('aria-label'),
+            placeholder: el.placeholder || null,
+            input_type: el.type || null,
+            href: (tag === 'a' && el.href) ? el.href.substring(0, 100) : null,
+            x: Math.round(rect.x),
+            y: Math.round(rect.y),
+            width: Math.round(rect.width),
+            height: Math.round(rect.height),
+            // Use injected attribute as locator - guaranteed to work!
+            locator: `[data-browser-agent-id="${highlightId}"]`
+        });
+    });
+
+    return results;
+})()

connectonion/cli/browser_agent/scroll.py

@@ -0,0 +1,145 @@
+"""
+Purpose: Intelligent page scrolling with AI strategy selection and fallback mechanisms
+LLM-Note:
+  Dependencies: imports from [playwright Page, connectonion llm_do, pydantic, pathlib, PIL Image] | imported by [cli/browser_agent/browser.py] | tested by [tests/cli/test_scroll.py]
+  Data flow: scroll(page, take_screenshot, times, description) → takes before screenshot → tries strategies: AI-generated JS → element scroll → page scroll → takes after screenshot → compares via _screenshots_different() → returns on first successful change | _ai_scroll() extracts scrollable elements + HTML → llm_do generates ScrollStrategy → executes JS | fallbacks use simpler JS approaches
+  State/Effects: writes before/after screenshots to screenshots/ directory | executes JavaScript on page (modifies scrollTop or window.scrollY) | no persistent state
+  Integration: exposes scroll(page, take_screenshot, times, description) → str | ScrollStrategy Pydantic model with method, selector, javascript, explanation | uses scroll_strategy.md prompt for LLM | verifies success via pixel difference (>1% change threshold)
+  Performance: tries AI first (slower but accurate) → falls back to faster heuristics | 0.5-1s sleep between scrolls for content loading | PIL pixel comparison (fast for typical screenshots)
+  Errors: returns "Browser not open" if page None | returns "All scroll strategies failed" if no strategy changes content | prints strategy attempts for debugging | catches exceptions per strategy and continues
+Unified scroll module - AI-powered with fallback strategies.
+
+Usage:
+    from scroll import scroll
+    result = scroll(page, take_screenshot, times=5, description="the email list")
+"""
+from pathlib import Path
+from pydantic import BaseModel
+from connectonion import llm_do
+import time
+
+_PROMPT = (Path(__file__).parent / "prompts" / "scroll_strategy.md").read_text()
+
+
+class ScrollStrategy(BaseModel):
+    method: str  # "window", "element", "container"
+    selector: str
+    javascript: str
+    explanation: str
+
+
+def scroll(page, take_screenshot, times: int = 5, description: str = "the main content area") -> str:
+    """Universal scroll with AI strategy and fallback.
+
+    Tries: AI-generated → Element scroll → Page scroll
+    Verifies success with screenshot comparison.
+    """
+    if not page:
+        return "Browser not open"
+
+    timestamp = int(time.time())
+    before = f"scroll_before_{timestamp}.png"
+    take_screenshot(path=before)
+
+    strategies = [
+        ("AI strategy", lambda: _ai_scroll(page, times, description)),
+        ("Element scroll", lambda: _element_scroll(page, times)),
+        ("Page scroll", lambda: _page_scroll(page, times)),
+    ]
+
+    for name, execute in strategies:
+        print(f"  Trying: {name}...")
+        try:
+            execute()
+            time.sleep(0.5)
+            after = f"scroll_after_{timestamp}.png"
+            take_screenshot(path=after)
+
+            if _screenshots_different(before, after):
+                print(f"  ✅ {name} worked")
+                return f"Scrolled using {name}"
+            print(f"  ⚠️ {name} didn't change content")
+            before = after
+        except Exception as e:
+            print(f"  ❌ {name} failed: {e}")
+
+    return "All scroll strategies failed"
+
+
+def _ai_scroll(page, times: int, description: str):
+    """AI-generated scroll strategy."""
+    scrollable = page.evaluate("""
+        (() => {
+            return Array.from(document.querySelectorAll('*'))
+                .filter(el => {
+                    const s = window.getComputedStyle(el);
+                    return (s.overflow === 'auto' || s.overflowY === 'scroll') &&
+                           el.scrollHeight > el.clientHeight;
+                })
+                .slice(0, 3)
+                .map(el => ({tag: el.tagName, classes: el.className, id: el.id}));
+        })()
+    """)
+
+    html = page.evaluate("""
+        (() => {
+            const c = document.body.cloneNode(true);
+            c.querySelectorAll('script,style,img,svg').forEach(e => e.remove());
+            return c.innerHTML.substring(0, 5000);
+        })()
+    """)
+
+    strategy = llm_do(
+        _PROMPT.format(description=description, scrollable_elements=scrollable, simplified_html=html),
+        output=ScrollStrategy,
+        model="co/gemini-2.5-flash",
+        temperature=0.1
+    )
+    print(f"    AI: {strategy.explanation}")
+
+    for _ in range(times):
+        page.evaluate(strategy.javascript)
+        time.sleep(1)
+
+
+def _element_scroll(page, times: int):
+    """Scroll first scrollable element found."""
+    for _ in range(times):
+        page.evaluate("""
+            (() => {
+                const el = Array.from(document.querySelectorAll('*')).find(e => {
+                    const s = window.getComputedStyle(e);
+                    return (s.overflow === 'auto' || s.overflowY === 'scroll') &&
+                           e.scrollHeight > e.clientHeight;
+                });
+                if (el) el.scrollTop += 1000;
+            })()
+        """)
+        time.sleep(0.8)
+
+
+def _page_scroll(page, times: int):
+    """Scroll window."""
+    for _ in range(times):
+        page.evaluate("window.scrollBy(0, 1000)")
+        time.sleep(0.8)
+
+
+def _screenshots_different(file1: str, file2: str) -> bool:
+    """Compare screenshots using PIL pixel difference."""
+    try:
+        from PIL import Image
+        import os
+
+        img1 = Image.open(os.path.join("screenshots", file1)).convert('RGB')
+        img2 = Image.open(os.path.join("screenshots", file2)).convert('RGB')
+
+        diff = sum(
+            abs(a - b)
+            for p1, p2 in zip(img1.getdata(), img2.getdata())
+            for a, b in zip(p1, p2)
+        )
+        threshold = img1.size[0] * img1.size[1] * 3 * 0.01  # 1%
+        return diff > threshold
+    except Exception:
+        return True  # Assume different if comparison fails

connectonion/cli/co_ai/__init__.py

@@ -0,0 +1,6 @@
+"""co-ai: AI coding agent built with ConnectOnion framework."""
+
+from connectonion.cli.co_ai.agent import create_coding_agent
+
+__version__ = "0.1.0"
+__all__ = ["create_coding_agent"]

connectonion/cli/co_ai/agent.py

@@ -0,0 +1,87 @@
+"""Main coding agent module."""
+
+from pathlib import Path
+
+from .context import load_project_context
+from .prompts.assembler import assemble_prompt
+from .tools import (
+    glob, grep, edit, read_file, task,
+    enter_plan_mode, exit_plan_mode, write_plan,
+    ask_user,
+    run_background, task_output, kill_task,
+    load_guide,
+)
+from .skills import skill
+from .plugins import reminder_plugin
+from connectonion import Agent, bash, after_user_input, FileWriter, MODE_AUTO, MODE_NORMAL, TodoList
+from connectonion.useful_plugins import re_act, eval
+
+
+PROMPTS_DIR = Path(__file__).parent / "prompts"
+
+
+@after_user_input
+def _sync_tool_io(agent) -> None:
+    io = getattr(agent, "io", None) or getattr(agent, "connection", None)
+    if not io:
+        return
+    # ToolRegistry does not expose instances publicly; use best-effort access.
+    instances = getattr(agent.tools, "_instances", {})
+    for instance in instances.values():
+        if hasattr(instance, "io"):
+            instance.io = io
+
+
+def create_coding_agent(
+    model: str = "co/claude-opus-4-5",
+    max_iterations: int = 20,
+    auto_approve: bool = False,
+    web_mode: bool = False,
+) -> Agent:
+    writer = FileWriter(mode=MODE_AUTO if auto_approve else MODE_NORMAL)
+    todo = TodoList()
+
+    tools = [
+        glob,
+        grep,
+        read_file,
+        edit,
+        writer,
+        bash,
+        task,
+        enter_plan_mode,
+        exit_plan_mode,
+        write_plan,
+        todo,
+        skill,
+        run_background,
+        task_output,
+        kill_task,
+        load_guide,
+        ask_user,
+    ]
+
+    base_prompt = assemble_prompt(
+        prompts_dir=str(PROMPTS_DIR),
+        tools=tools,
+    )
+
+    project_context = load_project_context()
+    system_prompt = base_prompt
+    if project_context:
+        system_prompt += f"\n\n---\n\n{project_context}"
+
+    plugins = [re_act, eval, reminder_plugin]
+
+    agent = Agent(
+        name="oo",
+        tools=tools,
+        plugins=plugins,
+        on_events=[_sync_tool_io],
+        system_prompt=system_prompt,
+        model=model,
+        max_iterations=max_iterations,
+    )
+
+    agent.writer = writer
+    return agent

connectonion/cli/co_ai/agents/__init__.py

@@ -0,0 +1,5 @@
+"""Sub-agents for OO."""
+
+from connectonion.cli.co_ai.agents.registry import SUBAGENTS, get_subagent
+
+__all__ = ["SUBAGENTS", "get_subagent"]

connectonion/cli/co_ai/agents/registry.py

@@ -0,0 +1,57 @@
+"""Sub-agent registry and factory."""
+
+from typing import Dict, Any, Optional
+from connectonion import Agent
+
+from connectonion.cli.co_ai.tools import glob, grep, read_file
+
+
+# Sub-agent configurations
+SUBAGENTS: Dict[str, Dict[str, Any]] = {
+    "explore": {
+        "description": "Fast agent for exploring codebases. Find files, search code, answer questions about structure.",
+        "tools": [glob, grep, read_file],
+        "model": "co/gemini-2.5-flash",  # Fast model for exploration
+        "max_iterations": 15,
+    },
+    "plan": {
+        "description": "Design implementation plans. Analyze architecture, identify files to change, plan steps.",
+        "tools": [glob, grep, read_file],
+        "model": "co/gemini-2.5-pro",  # Smarter model for planning
+        "max_iterations": 10,
+    },
+}
+
+
+def get_subagent(agent_type: str) -> Optional[Agent]:
+    """
+    Create a sub-agent instance by type.
+
+    Args:
+        agent_type: Type of sub-agent ("explore", "plan", etc.)
+
+    Returns:
+        Configured Agent instance or None if type not found
+    """
+    if agent_type not in SUBAGENTS:
+        return None
+
+    config = SUBAGENTS[agent_type]
+
+    # Load prompt from co_ai/prompts/agents/
+    from pathlib import Path
+    prompt_path = Path(__file__).parent.parent / "prompts" / "agents" / f"{agent_type}.md"
+
+    if prompt_path.exists():
+        system_prompt = prompt_path.read_text(encoding="utf-8")
+    else:
+        system_prompt = f"You are an {agent_type} agent. {config['description']}"
+
+    return Agent(
+        name=f"oo-{agent_type}",
+        tools=config["tools"],
+        plugins=[],
+        system_prompt=system_prompt,
+        model=config["model"],
+        max_iterations=config["max_iterations"],
+    )

connectonion/cli/co_ai/commands/__init__.py

@@ -0,0 +1,45 @@
+"""Built-in commands for OO CLI."""
+
+from connectonion.cli.co_ai.commands.init import cmd_init
+from connectonion.cli.co_ai.commands.help import cmd_help
+from connectonion.cli.co_ai.commands.cost import cmd_cost, set_agent as set_cost_agent
+from connectonion.cli.co_ai.commands.compact import cmd_compact, set_agent as set_compact_agent
+from connectonion.cli.co_ai.commands.tasks import cmd_tasks
+from connectonion.cli.co_ai.commands.export import cmd_export, set_agent as set_export_agent
+from connectonion.cli.co_ai.commands.sessions import cmd_sessions, cmd_new, cmd_resume, set_agent as set_sessions_agent
+from connectonion.cli.co_ai.commands.undo import cmd_undo, cmd_redo, set_agent as set_undo_agent
+
+BUILTIN_COMMANDS = {
+    "init": cmd_init,
+    "help": cmd_help,
+    "cost": cmd_cost,
+    "compact": cmd_compact,
+    "tasks": cmd_tasks,
+    "export": cmd_export,
+    "sessions": cmd_sessions,
+    "new": cmd_new,
+    "resume": cmd_resume,
+    "undo": cmd_undo,
+    "redo": cmd_redo,
+}
+
+
+def set_agent_for_commands(agent):
+    """Set agent reference for commands that need it."""
+    set_cost_agent(agent)
+    set_compact_agent(agent)
+    set_export_agent(agent)
+    set_sessions_agent(agent)
+    set_undo_agent(agent)
+
+
+__all__ = [
+    "BUILTIN_COMMANDS",
+    "cmd_init",
+    "cmd_help",
+    "cmd_cost",
+    "cmd_compact",
+    "cmd_tasks",
+    "cmd_export",
+    "set_agent_for_commands",
+]

connectonion/cli/co_ai/commands/compact.py

@@ -0,0 +1,173 @@
+"""Compact command for compressing conversation context using LLM summarization."""
+
+from pathlib import Path
+from rich.console import Console
+from rich.panel import Panel
+from connectonion import llm_do
+
+console = Console()
+
+# Module-level storage for agent reference
+_current_agent = None
+
+# Summarization prompt
+SUMMARIZATION_PROMPT = Path(__file__).parent.parent / "prompts" / "summarization.md"
+
+
+def set_agent(agent):
+    """Set the current agent for compact operation."""
+    global _current_agent
+    _current_agent = agent
+
+
+def _format_messages_for_summary(messages: list) -> str:
+    """Format messages into a readable format for summarization."""
+    formatted = []
+    for msg in messages:
+        role = msg.get('role', 'unknown')
+        content = msg.get('content', '')
+
+        # Handle tool_calls in assistant messages
+        if role == 'assistant' and 'tool_calls' in msg:
+            tool_calls = msg.get('tool_calls', [])
+            tools_str = ", ".join(tc.get('function', {}).get('name', 'unknown') for tc in tool_calls)
+            formatted.append(f"[assistant] Called tools: {tools_str}")
+            if content:
+                formatted.append(f"[assistant] {content[:500]}...")
+        elif role == 'tool':
+            # Truncate long tool results
+            tool_name = msg.get('name', 'unknown')
+            result = content[:200] + "..." if len(content) > 200 else content
+            formatted.append(f"[tool:{tool_name}] {result}")
+        elif role == 'system':
+            # Keep system messages brief
+            formatted.append(f"[system] {content[:300]}...")
+        else:
+            formatted.append(f"[{role}] {content}")
+
+    return "\n\n".join(formatted)
+
+
+def _create_summary_message(summary: str) -> dict:
+    """Create a summary message to replace old context."""
+    return {
+        'role': 'user',
+        'content': f"""## Previous Conversation Summary
+
+{summary}
+
+---
+*The conversation continues below. Use the summary above for context.*"""
+    }
+
+
+def cmd_compact(args: str = "") -> str:
+    """
+    Compress conversation history using LLM summarization.
+
+    This creates an intelligent summary of older messages while preserving:
+    - User's explicit requests and intents
+    - Key technical decisions and code patterns
+    - File names and important code snippets
+    - Errors encountered and how they were fixed
+    - Recent messages (last 5)
+
+    Use when context usage is high (>70%).
+    """
+    if _current_agent is None:
+        console.print("[yellow]No agent session active.[/]")
+        return "No session"
+
+    agent = _current_agent
+
+    # Check if we have a session
+    if not hasattr(agent, 'current_session') or not agent.current_session:
+        console.print("[yellow]No conversation history to compact.[/]")
+        return "No history"
+
+    messages = agent.current_session.get('messages', [])
+    if len(messages) < 8:
+        console.print("[yellow]Conversation too short to compact (< 8 messages).[/]")
+        return "Too short"
+
+    # Get context usage before
+    context_before = getattr(agent, 'context_percent', 0)
+
+    # Separate messages
+    system_msg = messages[0] if messages and messages[0].get('role') == 'system' else None
+    recent_count = 5
+    recent_msgs = messages[-recent_count:]
+    old_msgs = messages[1:-recent_count] if system_msg else messages[:-recent_count]
+
+    if len(old_msgs) < 3:
+        console.print("[yellow]Not enough old messages to compact.[/]")
+        return "Nothing to compact"
+
+    console.print("[cyan]Summarizing conversation...[/]")
+
+    # Load summarization prompt
+    summarization_instructions = ""
+    if SUMMARIZATION_PROMPT.exists():
+        summarization_instructions = SUMMARIZATION_PROMPT.read_text(encoding="utf-8")
+
+    # Format messages for summarization
+    conversation_text = _format_messages_for_summary(old_msgs)
+
+    # Use LLM to create intelligent summary
+    summary_prompt = f"""{summarization_instructions}
+
+## Conversation to Summarize
+
+{conversation_text}
+
+---
+
+Create a concise but complete summary following the structure above. Focus on:
+1. What the user wanted to accomplish
+2. Key files and code that were discussed or modified
+3. Any errors and how they were fixed
+4. Important decisions made
+
+Keep the summary under 1000 words but preserve all critical technical details."""
+
+    summary = llm_do(
+        summary_prompt,
+        model="co/gemini-2.5-flash",  # Fast model for summarization
+    )
+
+    # Create compacted messages
+    summary_message = _create_summary_message(summary)
+
+    new_messages = []
+    if system_msg:
+        new_messages.append(system_msg)
+    new_messages.append(summary_message)
+    new_messages.extend(recent_msgs)
+
+    # Update agent session
+    agent.current_session['messages'] = new_messages
+
+    # Calculate savings
+    old_count = len(messages)
+    new_count = len(new_messages)
+    saved = old_count - new_count
+
+    # Display result
+    console.print(Panel(
+        f"[green]✓ Conversation compacted[/]\n\n"
+        f"Messages: {old_count} → {new_count} ([green]-{saved}[/])\n"
+        f"Context before: {context_before:.0f}%\n\n"
+        f"[dim]Preserved:[/]\n"
+        f"  • System prompt\n"
+        f"  • AI-generated summary of older context\n"
+        f"  • Recent {recent_count} messages\n\n"
+        f"[dim]Summary includes:[/]\n"
+        f"  • User requests and intents\n"
+        f"  • Key files and code patterns\n"
+        f"  • Errors and fixes\n"
+        f"  • Technical decisions",
+        title="📦 Compact",
+        border_style="green"
+    ))
+
+    return f"Compacted: {old_count} → {new_count} messages"