connectonion 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

connectonion/cli/browser_agent/prompts/scroll_strategy.md

@@ -0,0 +1,36 @@
+# Scroll Strategy
+
+Analyze this webpage and determine the BEST way to scroll "{description}".
+
+## Scrollable Elements Found
+{scrollable_elements}
+
+## Simplified HTML (first 5000 chars)
+{simplified_html}
+
+## Instructions
+
+Return:
+
+1. **method**: "window" | "element" | "container"
+
+2. **selector**: CSS selector (empty if method is "window")
+
+3. **javascript**: Complete IIFE that scrolls ONE iteration:
+```javascript
+(() => {{
+  const el = document.querySelector('.selector');
+  if (el) el.scrollTop += 1000;
+  return {{success: true}};
+}})()
+```
+
+4. **explanation**: Brief reason
+
+## Common Patterns
+
+- Gmail/email lists: Scroll the container with overflow:auto, NOT window
+- Social feeds (Twitter, LinkedIn): Often scroll the main feed container
+- Regular pages: Use window.scrollBy(0, 1000)
+
+User wants to scroll: "{description}"

connectonion/cli/browser_agent/scripts/extract_elements.js

@@ -0,0 +1,126 @@
+/**
+ * Extract interactive elements from the page with injected IDs.
+ *
+ * Inspired by browser-use (https://github.com/browser-use/browser-use).
+ *
+ * This script:
+ * 1. Finds all interactive elements (buttons, links, inputs, etc.)
+ * 2. Injects a unique `data-browser-agent-id` attribute into each
+ * 3. Returns element data with bounding boxes for LLM matching
+ */
+(() => {
+    const results = [];
+    let index = 0;
+
+    // Interactive element types
+    const INTERACTIVE_TAGS = new Set([
+        'a', 'button', 'input', 'select', 'textarea', 'label',
+        'details', 'summary', 'dialog'
+    ]);
+
+    const INTERACTIVE_ROLES = new Set([
+        'button', 'link', 'menuitem', 'menuitemcheckbox', 'menuitemradio',
+        'option', 'radio', 'switch', 'tab', 'checkbox', 'textbox',
+        'searchbox', 'combobox', 'listbox', 'slider', 'spinbutton'
+    ]);
+
+    // Check if element is visible
+    function isVisible(el) {
+        const style = window.getComputedStyle(el);
+        if (style.display === 'none') return false;
+        if (style.visibility === 'hidden') return false;
+        if (parseFloat(style.opacity) === 0) return false;
+
+        // Skip visually-hidden accessibility elements
+        const className = el.className || '';
+        if (typeof className === 'string' &&
+            (className.includes('visually-hidden') ||
+             className.includes('sr-only') ||
+             className.includes('screen-reader'))) {
+            return false;
+        }
+
+        const rect = el.getBoundingClientRect();
+        if (rect.width === 0 || rect.height === 0) return false;
+
+        // Skip elements that are clipped/hidden with CSS tricks
+        if (rect.width < 2 || rect.height < 2) return false;
+
+        // Check if in viewport (with some margin)
+        const margin = 100;
+        if (rect.bottom < -margin) return false;
+        if (rect.top > window.innerHeight + margin) return false;
+        if (rect.right < -margin) return false;
+        if (rect.left > window.innerWidth + margin) return false;
+
+        return true;
+    }
+
+    // Get clean text content
+    function getText(el) {
+        // For inputs, get value or placeholder
+        if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
+            return el.value || el.placeholder || '';
+        }
+        // For other elements, get inner text
+        const text = el.innerText || el.textContent || '';
+        return text.trim().replace(/\s+/g, ' ').substring(0, 80);
+    }
+
+    // Process all elements
+    document.querySelectorAll('*').forEach(el => {
+        const tag = el.tagName.toLowerCase();
+        const role = el.getAttribute('role');
+
+        // Check if interactive
+        const isInteractiveTag = INTERACTIVE_TAGS.has(tag);
+        const isInteractiveRole = role && INTERACTIVE_ROLES.has(role);
+        const isClickable = window.getComputedStyle(el).cursor === 'pointer';
+        const hasTabIndex = el.hasAttribute('tabindex') && el.tabIndex >= 0;
+        const hasClickHandler = el.onclick !== null || el.hasAttribute('onclick');
+
+        if (!isInteractiveTag && !isInteractiveRole && !isClickable &&
+            !hasTabIndex && !hasClickHandler) {
+            return;
+        }
+
+        // Skip hidden inputs
+        if (tag === 'input' && el.type === 'hidden') return;
+
+        // Skip empty elements with no text or useful attributes
+        const text = getText(el);
+        const ariaLabel = el.getAttribute('aria-label');
+        const placeholder = el.placeholder;
+        if (!text && !ariaLabel && !placeholder && tag !== 'input') return;
+
+        // Skip very small elements (likely icons)
+        const rect = el.getBoundingClientRect();
+        if (rect.width < 20 && rect.height < 20 && !text) return;
+
+        // Check visibility
+        if (!isVisible(el)) return;
+
+        // INJECT a unique ID attribute for reliable location
+        const highlightId = String(index);
+        el.setAttribute('data-browser-agent-id', highlightId);
+
+        results.push({
+            index: index++,
+            tag: tag,
+            text: text,
+            role: role,
+            aria_label: el.getAttribute('aria-label'),
+            placeholder: el.placeholder || null,
+            input_type: el.type || null,
+            href: (tag === 'a' && el.href) ? el.href.substring(0, 100) : null,
+            x: Math.round(rect.x),
+            y: Math.round(rect.y),
+            width: Math.round(rect.width),
+            height: Math.round(rect.height),
+            // Use injected attribute as locator - guaranteed to work!
+            locator: `[data-browser-agent-id="${highlightId}"]`
+        });
+    });
+
+    return results;
+})()

connectonion/cli/browser_agent/scroll.py

@@ -0,0 +1,137 @@
+"""
+Unified scroll module - AI-powered with fallback strategies.
+
+Usage:
+    from scroll import scroll
+    result = scroll(page, take_screenshot, times=5, description="the email list")
+"""
+from pathlib import Path
+from pydantic import BaseModel
+from connectonion import llm_do
+import time
+
+_PROMPT = (Path(__file__).parent / "prompts" / "scroll_strategy.md").read_text()
+
+
+class ScrollStrategy(BaseModel):
+    method: str  # "window", "element", "container"
+    selector: str
+    javascript: str
+    explanation: str
+
+
+def scroll(page, take_screenshot, times: int = 5, description: str = "the main content area") -> str:
+    """Universal scroll with AI strategy and fallback.
+
+    Tries: AI-generated → Element scroll → Page scroll
+    Verifies success with screenshot comparison.
+    """
+    if not page:
+        return "Browser not open"
+
+    timestamp = int(time.time())
+    before = f"scroll_before_{timestamp}.png"
+    take_screenshot(path=before)
+
+    strategies = [
+        ("AI strategy", lambda: _ai_scroll(page, times, description)),
+        ("Element scroll", lambda: _element_scroll(page, times)),
+        ("Page scroll", lambda: _page_scroll(page, times)),
+    ]
+
+    for name, execute in strategies:
+        print(f"  Trying: {name}...")
+        try:
+            execute()
+            time.sleep(0.5)
+            after = f"scroll_after_{timestamp}.png"
+            take_screenshot(path=after)
+
+            if _screenshots_different(before, after):
+                print(f"  ✅ {name} worked")
+                return f"Scrolled using {name}"
+            print(f"  ⚠️ {name} didn't change content")
+            before = after
+        except Exception as e:
+            print(f"  ❌ {name} failed: {e}")
+
+    return "All scroll strategies failed"
+
+
+def _ai_scroll(page, times: int, description: str):
+    """AI-generated scroll strategy."""
+    scrollable = page.evaluate("""
+        (() => {
+            return Array.from(document.querySelectorAll('*'))
+                .filter(el => {
+                    const s = window.getComputedStyle(el);
+                    return (s.overflow === 'auto' || s.overflowY === 'scroll') &&
+                           el.scrollHeight > el.clientHeight;
+                })
+                .slice(0, 3)
+                .map(el => ({tag: el.tagName, classes: el.className, id: el.id}));
+        })()
+    """)
+
+    html = page.evaluate("""
+        (() => {
+            const c = document.body.cloneNode(true);
+            c.querySelectorAll('script,style,img,svg').forEach(e => e.remove());
+            return c.innerHTML.substring(0, 5000);
+        })()
+    """)
+
+    strategy = llm_do(
+        _PROMPT.format(description=description, scrollable_elements=scrollable, simplified_html=html),
+        output=ScrollStrategy,
+        model="co/gemini-2.5-flash",
+        temperature=0.1
+    )
+    print(f"    AI: {strategy.explanation}")
+
+    for _ in range(times):
+        page.evaluate(strategy.javascript)
+        time.sleep(1)
+
+
+def _element_scroll(page, times: int):
+    """Scroll first scrollable element found."""
+    for _ in range(times):
+        page.evaluate("""
+            (() => {
+                const el = Array.from(document.querySelectorAll('*')).find(e => {
+                    const s = window.getComputedStyle(e);
+                    return (s.overflow === 'auto' || s.overflowY === 'scroll') &&
+                           e.scrollHeight > e.clientHeight;
+                });
+                if (el) el.scrollTop += 1000;
+            })()
+        """)
+        time.sleep(0.8)
+
+
+def _page_scroll(page, times: int):
+    """Scroll window."""
+    for _ in range(times):
+        page.evaluate("window.scrollBy(0, 1000)")
+        time.sleep(0.8)
+
+
+def _screenshots_different(file1: str, file2: str) -> bool:
+    """Compare screenshots using PIL pixel difference."""
+    try:
+        from PIL import Image
+        import os
+
+        img1 = Image.open(os.path.join("screenshots", file1)).convert('RGB')
+        img2 = Image.open(os.path.join("screenshots", file2)).convert('RGB')
+
+        diff = sum(
+            abs(a - b)
+            for p1, p2 in zip(img1.getdata(), img2.getdata())
+            for a, b in zip(p1, p2)
+        )
+        threshold = img1.size[0] * img1.size[1] * 3 * 0.01  # 1%
+        return diff > threshold
+    except Exception:
+        return True  # Assume different if comparison fails

connectonion/cli/commands/eval_commands.py

@@ -0,0 +1,286 @@
+"""
+Purpose: CLI command for running and managing evals
+LLM-Note:
+  Dependencies: imports from [pathlib, yaml, json, rich, importlib] | imported by [cli/main.py]
+  Data flow: handle_eval() → reads .co/evals/*.yaml → imports agent → runs with stored input → compares expected vs output
+  Integration: exposes handle_eval(name, run) for CLI
+
+Eval YAML format:
+  - `turns`: List of inputs to send to agent sequentially (like a conversation).
+    Each turn can have one input. Turns run in order within same agent session,
+    simulating multi-round conversations. Use single turn for simple evals,
+    or multiple turns to test conversation flow.
+"""
+
+import importlib.util
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import yaml
+from pydantic import BaseModel
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+class JudgeResult(BaseModel):
+    """Result from LLM judge evaluation."""
+    passed: bool
+    analysis: str
+
+
+def get_agent_from_file(file_path: str, cwd: str):
+    """Import agent instance from file."""
+    from connectonion import Agent
+
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(cwd, file_path)
+
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+
+    spec = importlib.util.spec_from_file_location("agent_module", file_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    if hasattr(module, 'agent') and isinstance(module.agent, Agent):
+        agent = module.agent
+        agent.logger.enable_sessions = False  # Prevent duplicate eval files
+        return agent
+
+    raise ValueError(
+        f"No 'agent' instance found in {file_path}.\n\n"
+        f"Structure your file like this:\n\n"
+        f"    agent = Agent(...)\n\n"
+        f"    if __name__ == '__main__':\n"
+        f"        agent.input('...')\n"
+    )
+
+
+def handle_eval(name: Optional[str] = None, agent_file: Optional[str] = None):
+    """Run evals and show results.
+
+    Args:
+        name: Optional specific eval name to run
+        agent_file: Optional agent file path (overrides YAML setting)
+    """
+    evals_dir = Path(".co/evals")
+
+    if not evals_dir.exists():
+        console.print("[yellow]No evals found.[/yellow]")
+        console.print("[dim]Create eval files in .co/evals/*.yaml[/dim]")
+        return
+
+    if name:
+        eval_files = list(evals_dir.glob(f"{name}.yaml"))
+        if not eval_files:
+            console.print(f"[red]Eval not found: {name}[/red]")
+            return
+    else:
+        eval_files = list(evals_dir.glob("*.yaml"))
+
+    if not eval_files:
+        console.print("[yellow]No eval files found in .co/evals/[/yellow]")
+        return
+
+    _run_evals(eval_files, agent_file)
+
+    # Reload and show status
+    if name:
+        eval_files = list(evals_dir.glob(f"{name}.yaml"))
+    else:
+        eval_files = list(evals_dir.glob("*.yaml"))
+
+    _show_eval_status(eval_files)
+
+
+def _run_evals(eval_files: list, agent_override: Optional[str] = None):
+    """Run agents for each eval and capture output."""
+    cwd = os.getcwd()
+    agents_cache = {}  # Cache agents by file path
+
+    for eval_file in eval_files:
+        with open(eval_file) as f:
+            data = yaml.safe_load(f)
+
+        # Get agent file: CLI override > YAML > error
+        agent_file = agent_override or data.get('agent')
+        if not agent_file:
+            console.print(f"[red]No agent specified for {eval_file.stem}[/red]")
+            console.print(f"[dim]Add 'agent: agent.py' to the YAML or use --agent flag[/dim]")
+            continue
+
+        # Load agent (cached)
+        if agent_file not in agents_cache:
+            console.print(f"[cyan]Loading:[/cyan] {agent_file}")
+            agents_cache[agent_file] = get_agent_from_file(agent_file, cwd)
+        agent = agents_cache[agent_file]
+
+        turns = data.get('turns', [])
+        if not turns:
+            console.print(f"[yellow]No turns found in {eval_file.stem}[/yellow]")
+            continue
+
+        console.print(f"[cyan]Running:[/cyan] {eval_file.stem}")
+
+        # Reset agent session for fresh state each eval
+        agent.reset_conversation()
+
+        file_modified = False
+        for turn in turns:
+            input_text = turn.get('input', '')
+            if not input_text:
+                continue
+
+            # Show input (truncated)
+            display_input = input_text[:60] + "..." if len(input_text) > 60 else input_text
+            console.print(f"  [dim]input:[/dim] {display_input}")
+
+            # Run agent and capture result
+            result = agent.input(input_text)
+
+            # Extract tools_called and metrics from agent session
+            trace = agent.current_session.get('trace', [])
+            tool_calls = [t for t in trace if t.get('type') == 'tool_execution']
+            llm_calls = [t for t in trace if t.get('type') == 'llm_call']
+            tools_called = [agent.logger._format_tool_call(t) for t in tool_calls]
+
+            total_tokens = sum(
+                (t.get('usage').input_tokens + t.get('usage').output_tokens)
+                for t in llm_calls if t.get('usage')
+            )
+            total_cost = sum(
+                t.get('usage').cost for t in llm_calls if t.get('usage')
+            )
+
+            # Build history as JSON array string (compact, easy to scan)
+            history_str = turn.get('history', '[]')
+            history = json.loads(history_str) if isinstance(history_str, str) else []
+            if turn.get('output'):
+                history.insert(0, {
+                    "ts": turn.get('ts', ''),
+                    "pass": turn.get('pass'),
+                    "tokens": turn.get('tokens', 0),
+                    "cost": turn.get('cost', 0)
+                })
+
+            # Store result in turn
+            turn['output'] = result
+            turn['tools_called'] = tools_called
+            turn['tokens'] = total_tokens
+            turn['cost'] = round(total_cost, 4)
+            turn['ts'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            turn['run'] = data.get('runs', 0) + 1
+            # Format history as multi-line JSON for readability
+            if history:
+                lines = [json.dumps(h) for h in history]
+                turn['history'] = "[\n" + ",\n".join(lines) + "]"
+            else:
+                turn['history'] = "[]"
+            file_modified = True
+
+            # Judge immediately if expected exists
+            expected = turn.get('expected', '')
+            if expected:
+                judge = _judge_with_llm(expected, result, input_text)
+                turn['pass'] = judge.passed
+                turn['analysis'] = judge.analysis
+                status = "[green]✓[/green]" if judge.passed else "[red]✗[/red]"
+                console.print(f"  {status} {judge.analysis[:60]}...")
+            else:
+                # Show output (truncated)
+                display_output = result[:60] + "..." if len(result) > 60 else result
+                console.print(f"  [green]output:[/green] {display_output}")
+
+        if file_modified:
+            # Update runs count and save
+            data['runs'] = data.get('runs', 0) + 1
+            data['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            with open(eval_file, 'w') as f:
+                yaml.dump(data, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
+
+        console.print(f"[green]✓[/green] {eval_file.stem} completed")
+        console.print()
+
+    console.print()
+
+
+def _judge_with_llm(expected: str, output: str, input_text: str) -> JudgeResult:
+    """Use LLM to judge if output matches expected."""
+    from connectonion import llm_do
+
+    prompt = f"""You are an eval judge. Determine if the agent's output satisfies the expected criteria.
+
+Input: {input_text}
+Expected: {expected}
+Output: {output}
+
+Does the output satisfy the expected criteria? Consider:
+- Semantic similarity (not exact match)
+- Key information presence
+- Intent fulfillment
+"""
+    return llm_do(prompt, output=JudgeResult)
+
+
+def _show_eval_status(eval_files: list):
+    """Show pass/fail status for all evals (uses stored results, no re-judging)."""
+    table = Table(title="Eval Results", show_header=True)
+    table.add_column("Eval", style="cyan")
+    table.add_column("Status", justify="center")
+    table.add_column("Expected", max_width=30)
+    table.add_column("Output", max_width=30)
+
+    passed = 0
+    failed = 0
+    no_expected = 0
+
+    for eval_file in sorted(eval_files):
+        with open(eval_file) as f:
+            data = yaml.safe_load(f)
+
+        for turn in data.get('turns', []):
+            expected = turn.get('expected', '')
+            output = turn.get('output', '')
+            pass_result = turn.get('pass')
+
+            if not expected:
+                status = "[dim]—[/dim]"
+                no_expected += 1
+            elif pass_result is True:
+                status = "[green]✓ pass[/green]"
+                passed += 1
+            elif pass_result is False:
+                status = "[red]✗ fail[/red]"
+                failed += 1
+            else:
+                status = "[dim]pending[/dim]"
+                no_expected += 1
+
+            # Truncate for display
+            expected_display = (expected[:27] + "...") if len(expected) > 30 else expected
+            output_display = (output[:27] + "...") if len(output) > 30 else output
+
+            table.add_row(
+                eval_file.stem,
+                status,
+                expected_display or "[dim]not set[/dim]",
+                output_display
+            )
+
+    console.print(table)
+    console.print()
+
+    # Summary
+    if failed > 0:
+        console.print(f"[bold red]✗ {failed} failed[/bold red], ", end="")
+    if passed > 0:
+        console.print(f"[bold green]✓ {passed} passed[/bold green], ", end="")
+    if no_expected > 0:
+        console.print(f"[dim]{no_expected} no expected[/dim]", end="")
+    console.print()

connectonion/cli/main.py

@@ -55,6 +55,7 @@ def _show_help():
     console.print("  [green]create[/green]  <name>     Create new project")
     console.print("  [green]init[/green]              Initialize in current directory")
     console.print("  [green]copy[/green]   <name>     Copy tool/plugin source to project")
+    console.print("  [green]eval[/green]              Run evals and show status")
     console.print("  [green]deploy[/green]            Deploy to ConnectOnion Cloud")
     console.print("  [green]auth[/green]              Authenticate for managed keys")
     console.print("  [green]status[/green]            Check account balance")
@@ -152,6 +153,16 @@ def copy(
     handle_copy(names=names or [], list_all=list_all, path=path, force=force)
 
 
+@app.command()
+def eval(
+    name: Optional[str] = typer.Argument(None, help="Specific eval name"),
+    agent: Optional[str] = typer.Option(None, "--agent", "-a", help="Agent file (overrides YAML)"),
+):
+    """Run evals and show results."""
+    from .commands.eval_commands import handle_eval
+    handle_eval(name=name, agent_file=agent)
+
+
 def cli():
     """Entry point."""
     app()

connectonion/console.py

@@ -110,7 +110,7 @@ class Console:
         ●     ─────────────────────
               connectonion v0.5.1
               o4-mini · 3 tools
-              .co/logs/ · .co/sessions/
+              .co/logs/ · .co/evals/
 
         Args:
             agent_name: Name of the agent
@@ -156,7 +156,7 @@ class Console:
 
         # Add log paths if logging is enabled
         if log_dir:
-            lines.append(f"      [{DIM_COLOR}]{log_dir}logs/ · {log_dir}sessions/[/{DIM_COLOR}]")
+            lines.append(f"      [{DIM_COLOR}]{log_dir}logs/ · {log_dir}evals/[/{DIM_COLOR}]")
 
         # Add Aaron's message for free tier users
         if aaron_message:
@@ -182,7 +182,7 @@ class Console:
             if meta_line:
                 plain_lines.append(f"      {meta_line}")
             if log_dir:
-                plain_lines.append(f"      {log_dir}logs/ · {log_dir}sessions/")
+                plain_lines.append(f"      {log_dir}logs/ · {log_dir}evals/")
             if aaron_message:
                 plain_lines.append(f"      {aaron_message}")
             plain_lines.append(f"      {separator}")
@@ -484,12 +484,12 @@ class Console:
 
         [co] ═══════════════════════════════════════
         [co] ✓ done · 2.3k tokens · $0.005 · 3.4s
-        [co]   saved → .co/sessions/research-assistant.yaml
+        [co]   saved → .co/evals/research-assistant.yaml
 
         Args:
             duration_s: Total duration in seconds
             session: Agent's current_session dict (contains trace with usage)
-            session_path: Optional path to session file
+            session_path: Optional path to eval file
         """
         # Calculate totals from trace
         trace = session.get('trace', [])