camel-ai 0.2.75a6__py3-none-any.whl → 0.2.76a0__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts

@@ -2,18 +2,22 @@ import {HybridBrowserSession} from './browser-session';
 import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';
 import {ConfigLoader} from './config-loader';
 import {ConsoleMessage} from 'playwright';
+import {SomScreenshotInjected} from './som-screenshot-injected';
+import {filterClickableByHierarchy} from './snapshot-parser';
 
 export class HybridBrowserToolkit {
   private session: HybridBrowserSession;
   private config: BrowserToolkitConfig;
   private configLoader: ConfigLoader;
   private viewportLimit: boolean;
+  private fullVisualMode: boolean;
 
   constructor(config: BrowserToolkitConfig = {}) {
     this.configLoader = ConfigLoader.fromPythonConfig(config);
     this.config = config; // Store original config for backward compatibility
     this.session = new HybridBrowserSession(this.configLoader.getBrowserConfig()); // Pass processed config
     this.viewportLimit = this.configLoader.getWebSocketConfig().viewport_limit;
+    this.fullVisualMode = this.configLoader.getWebSocketConfig().fullVisualMode || false;
   }
 
   async openBrowser(startUrl?: string): Promise<ActionResult> {
@@ -26,7 +30,7 @@ export class HybridBrowserToolkit {
       const result = await this.session.visitPage(url);
       
       const snapshotStart = Date.now();
-      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshot = await this.getSnapshotForAction(this.viewportLimit);
       const snapshotTime = Date.now() - snapshotStart;
       
       const totalTime = Date.now() - startTime;
@@ -83,7 +87,7 @@ export class HybridBrowserToolkit {
       
       if (result.success) {
         const snapshotStart = Date.now();
-        response.snapshot = await this.getPageSnapshot(this.viewportLimit);
+        response.snapshot = await this.getSnapshotForAction(this.viewportLimit);
         const snapshotTime = Date.now() - snapshotStart;
         
         if (result.timing) {
@@ -119,6 +123,7 @@ export class HybridBrowserToolkit {
 
   async getPageSnapshot(viewportLimit: boolean = false): Promise<string> {
     try {
+      // Always return real snapshot when explicitly called
       // If viewport limiting is enabled, we need coordinates for filtering
       const snapshotResult = await this.session.getSnapshotForAI(viewportLimit, viewportLimit);
       return snapshotResult.snapshot;
@@ -126,6 +131,14 @@ export class HybridBrowserToolkit {
       return `Error capturing snapshot: ${error}`;
     }
   }
+  
+  // Internal method for getting snapshot in actions (respects fullVisualMode)
+  private async getSnapshotForAction(viewportLimit: boolean = false): Promise<string> {
+    if (this.fullVisualMode) {
+      return 'full visual mode';
+    }
+    return this.getPageSnapshot(viewportLimit);
+  }
 
 
   async getSnapshotForAI(): Promise<SnapshotResult> {
@@ -134,35 +147,34 @@ export class HybridBrowserToolkit {
 
   async getSomScreenshot(): Promise<VisualMarkResult & { timing: any }> {
     const startTime = Date.now();
+    console.log('[HybridBrowserToolkit] Starting getSomScreenshot...');
     
     try {
-      const screenshotResult = await this.session.takeScreenshot();
-      const snapshotResult = await this.session.getSnapshotForAI(true); // Include coordinates for SOM_mark
-      
-      // Add visual marks using improved method
-      const markingStart = Date.now();
-      const markedImageBuffer = await this.addVisualMarksOptimized(screenshotResult.buffer, snapshotResult);
-      const markingTime = Date.now() - markingStart;
+      // Get page and snapshot data
+      const page = await this.session.getCurrentPage();
+      const snapshotResult = await this.session.getSnapshotForAI(true); // Include coordinates
       
-      const base64Image = markedImageBuffer.toString('base64');
-      const dataUrl = `data:image/png;base64,${base64Image}`;
+      // Parse clickable elements from snapshot text
+      const clickableElements = this.parseClickableElements(snapshotResult.snapshot);
+      console.log(`[HybridBrowserToolkit] Found ${clickableElements.size} clickable elements`);
       
-      const totalTime = Date.now() - startTime;
+      // Apply hierarchy-based filtering
+      const filteredElements = filterClickableByHierarchy(snapshotResult.snapshot, clickableElements);
+      console.log(`[HybridBrowserToolkit] After filtering: ${filteredElements.size} elements remain`);
+
+      // Use injected SOM-screenshot method without export path
+      const result = await SomScreenshotInjected.captureOptimized(
+        page,
+        snapshotResult,
+        filteredElements,
+        undefined  // No export path - don't generate files
+      );
       
-      // Count elements with coordinates
-      const elementsWithCoords = Object.values(snapshotResult.elements).filter(el => el.coordinates).length;
+      // Add snapshot timing info to result
+      result.timing.snapshot_time_ms = snapshotResult.timing.snapshot_time_ms;
+      result.timing.coordinate_enrichment_time_ms = snapshotResult.timing.coordinate_enrichment_time_ms;
       
-      return {
-        text: `Visual webpage screenshot captured with ${Object.keys(snapshotResult.elements).length} interactive elements (${elementsWithCoords} marked visually)`,
-        images: [dataUrl],
-        timing: {
-          total_time_ms: totalTime,
-          screenshot_time_ms: screenshotResult.timing.screenshot_time_ms,
-          snapshot_time_ms: snapshotResult.timing.snapshot_time_ms,
-          coordinate_enrichment_time_ms: snapshotResult.timing.coordinate_enrichment_time_ms,
-          visual_marking_time_ms: markingTime,
-        },
-      };
+      return result;
     } catch (error) {
       const totalTime = Date.now() - startTime;
       return {
@@ -179,132 +191,6 @@ export class HybridBrowserToolkit {
     }
   }
 
-  private async addVisualMarksOptimized(screenshotBuffer: Buffer, snapshotResult: SnapshotResult): Promise<Buffer> {
-    try {
-      
-      // Check if we have any elements with coordinates
-      const elementsWithCoords = Object.entries(snapshotResult.elements)
-        .filter(([ref, element]) => element.coordinates);
-      
-      if (elementsWithCoords.length === 0) {
-        return screenshotBuffer;
-      }
-      
-      // Parse clickable elements from snapshot text
-      const clickableElements = this.parseClickableElements(snapshotResult.snapshot);
-      
-      // Use sharp for image processing
-      const sharp = require('sharp');
-      const page = await this.session.getCurrentPage();
-      let viewport = page.viewportSize();
-      
-      // In CDP mode, viewportSize might be null, get it from window dimensions
-      if (!viewport) {
-        const windowSize = await page.evaluate(() => ({
-          width: window.innerWidth,
-          height: window.innerHeight
-        }));
-        viewport = windowSize;
-      }
-      
-      // Get device pixel ratio to handle high DPI screens
-      const dpr = await page.evaluate(() => window.devicePixelRatio) || 1;
-      
-      // Get actual screenshot dimensions
-      const metadata = await sharp(screenshotBuffer).metadata();
-      const screenshotWidth = metadata.width || viewport.width;
-      const screenshotHeight = metadata.height || viewport.height;
-      
-      // Calculate scaling factor between CSS pixels and screenshot pixels
-      const scaleX = screenshotWidth / viewport.width;
-      const scaleY = screenshotHeight / viewport.height;
-      
-      // Debug logging for CDP mode
-      if (process.env.HYBRID_BROWSER_DEBUG === '1') {
-        console.log('[CDP Debug] Viewport size:', viewport);
-        console.log('[CDP Debug] Device pixel ratio:', dpr);
-        console.log('[CDP Debug] Screenshot dimensions:', { width: screenshotWidth, height: screenshotHeight });
-        console.log('[CDP Debug] Scale factors:', { scaleX, scaleY });
-        console.log('[CDP Debug] Elements with coordinates:', elementsWithCoords.length);
-        elementsWithCoords.slice(0, 3).forEach(([ref, element]) => {
-          console.log(`[CDP Debug] Element ${ref}:`, element.coordinates);
-        });
-      }
-      
-      // Filter elements visible in viewport
-      const visibleElements = elementsWithCoords.filter(([ref, element]) => {
-        const coords = element.coordinates!;
-        return coords.x < viewport.width && 
-               coords.y < viewport.height && 
-               coords.x + coords.width > 0 && 
-               coords.y + coords.height > 0;
-      });
-      
-      // Remove overlapped elements (only keep topmost)
-      const nonOverlappedElements = this.removeOverlappedElements(visibleElements);
-      
-      // Create SVG overlay with all the marks
-      const marks = nonOverlappedElements.map(([ref, element]) => {
-        const coords = element.coordinates!;
-        const isClickable = clickableElements.has(ref);
-        
-        // Scale coordinates from CSS pixels to screenshot pixels
-        const x = Math.max(0, coords.x * scaleX);
-        const y = Math.max(0, coords.y * scaleY);
-        const width = coords.width * scaleX;
-        const height = coords.height * scaleY;
-        
-        // Clamp to screenshot bounds
-        const clampedWidth = Math.min(width, screenshotWidth - x);
-        const clampedHeight = Math.min(height, screenshotHeight - y);
-        
-        // Position text to be visible even if element is partially cut off
-        const textX = Math.max(2, Math.min(x + 2, screenshotWidth - 40));
-        const textY = Math.max(14, Math.min(y + 14, screenshotHeight - 4));
-        
-        // Different colors for clickable vs non-clickable elements
-        const colors = isClickable ? {
-          fill: 'rgba(0, 150, 255, 0.15)',  // Blue for clickable
-          stroke: '#0096FF',
-          textFill: '#0096FF'
-        } : {
-          fill: 'rgba(255, 107, 107, 0.1)',  // Red for non-clickable
-          stroke: '#FF6B6B', 
-          textFill: '#FF6B6B'
-        };
-        
-        return `
-          <rect x="${x}" y="${y}" width="${clampedWidth}" height="${clampedHeight}" 
-                fill="${colors.fill}" stroke="${colors.stroke}" stroke-width="2" rx="2"/>
-          <text x="${textX}" y="${textY}" font-family="Arial, sans-serif" 
-                font-size="12" fill="${colors.textFill}" font-weight="bold">${ref}</text>
-        `;
-      }).join('');
-      
-      const svgOverlay = `
-        <svg width="${screenshotWidth}" height="${screenshotHeight}" xmlns="http://www.w3.org/2000/svg">
-          ${marks}
-        </svg>
-      `;
-      
-      // Composite the overlay onto the screenshot
-      const markedImageBuffer = await sharp(screenshotBuffer)
-        .composite([{
-          input: Buffer.from(svgOverlay),
-          top: 0,
-          left: 0
-        }])
-        .png()
-        .toBuffer();
-      
-      return markedImageBuffer;
-      
-    } catch (error) {
-      // Error adding visual marks, falling back to original screenshot
-      // Return original screenshot if marking fails
-      return screenshotBuffer;
-    }
-  }
 
   /**
    * Parse clickable elements from snapshot text
@@ -314,8 +200,8 @@ export class HybridBrowserToolkit {
     const lines = snapshotText.split('\n');
     
     for (const line of lines) {
-      // Look for lines containing [cursor=pointer] and extract ref
-      if (line.includes('[cursor=pointer]')) {
+      // Look for lines containing [cursor=pointer] or [active] and extract ref
+      if (line.includes('[cursor=pointer]') || line.includes('[active]')) {
         const refMatch = line.match(/\[ref=([^\]]+)\]/);
         if (refMatch) {
           clickableElements.add(refMatch[1]);
@@ -326,56 +212,6 @@ export class HybridBrowserToolkit {
     return clickableElements;
   }
 
-  /**
-   * Remove overlapped elements, keeping only the topmost (last in DOM order)
-   */
-  private removeOverlappedElements(elements: Array<[string, any]>): Array<[string, any]> {
-    const result: Array<[string, any]> = [];
-    
-    for (let i = 0; i < elements.length; i++) {
-      const [refA, elementA] = elements[i];
-      const coordsA = elementA.coordinates!;
-      let isOverlapped = false;
-      
-      // Check if this element is completely overlapped by any later element
-      for (let j = i + 1; j < elements.length; j++) {
-        const [refB, elementB] = elements[j];
-        const coordsB = elementB.coordinates!;
-        
-        // Check if element A is completely covered by element B
-        if (this.isCompletelyOverlapped(coordsA, coordsB)) {
-          isOverlapped = true;
-          break;
-        }
-      }
-      
-      if (!isOverlapped) {
-        result.push(elements[i]);
-      }
-    }
-    
-    return result;
-  }
-
-  /**
-   * Check if element A is completely overlapped by element B
-   */
-  private isCompletelyOverlapped(
-    coordsA: { x: number; y: number; width: number; height: number },
-    coordsB: { x: number; y: number; width: number; height: number }
-  ): boolean {
-    // A is completely overlapped by B if:
-    // B's left edge is <= A's left edge AND
-    // B's top edge is <= A's top edge AND  
-    // B's right edge is >= A's right edge AND
-    // B's bottom edge is >= A's bottom edge
-    return (
-      coordsB.x <= coordsA.x &&
-      coordsB.y <= coordsA.y &&
-      coordsB.x + coordsB.width >= coordsA.x + coordsA.width &&
-      coordsB.y + coordsB.height >= coordsA.y + coordsA.height
-    );
-  }
 
   private async executeActionWithSnapshot(action: BrowserAction): Promise<any> {
     const result = await this.session.executeAction(action);
@@ -472,7 +308,7 @@ export class HybridBrowserToolkit {
       const navigationTime = Date.now() - navigationStart;
       
       const snapshotStart = Date.now();
-      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshot = await this.getSnapshotForAction(this.viewportLimit);
       const snapshotTime = Date.now() - snapshotStart;
       
       const totalTime = Date.now() - startTime;
@@ -512,7 +348,7 @@ export class HybridBrowserToolkit {
       const navigationTime = Date.now() - navigationStart;
       
       const snapshotStart = Date.now();
-      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshot = await this.getSnapshotForAction(this.viewportLimit);
       const snapshotTime = Date.now() - snapshotStart;
       
       const totalTime = Date.now() - startTime;
@@ -584,7 +420,7 @@ export class HybridBrowserToolkit {
       return {
         success: true,
         message: `Closed tab ${tabId}`,
-        snapshot: await this.getPageSnapshot(this.viewportLimit),
+        snapshot: await this.getSnapshotForAction(this.viewportLimit),
       };
     } else {
       return {
@@ -649,7 +485,7 @@ export class HybridBrowserToolkit {
       const { result, logs } = evalResult;
 
       const snapshotStart = Date.now();
-      const snapshot = await this.getPageSnapshot(this.viewportLimit);
+      const snapshot = await this.getSnapshotForAction(this.viewportLimit);
       const snapshotTime = Date.now() - snapshotStart;
       const totalTime = Date.now() - startTime;
 

camel/toolkits/hybrid_browser_toolkit/ts/src/parent-child-filter.ts

@@ -0,0 +1,226 @@
+/**
+ * Parent-child filtering logic for SOM-labels
+ * Filters out child elements that are contained within propagating parent elements
+ */
+
+export interface ElementInfo {
+  ref: string;
+  coordinates?: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+  };
+  role?: string;
+  type?: string;
+  tagName?: string;
+  attributes?: Record<string, any>;
+  text?: string;
+}
+
+// Elements that propagate bounds to their children
+const PROPAGATING_ELEMENTS = [
+  { tag: 'a', role: null },
+  { tag: 'button', role: null },
+  { tag: 'div', role: 'button' },
+  { tag: 'div', role: 'combobox' },
+  { tag: 'span', role: 'button' },
+  { tag: 'span', role: 'combobox' },
+  { tag: 'input', role: 'combobox' },
+];
+
+const CONTAINMENT_THRESHOLD = 0.99; // 99% containment required
+
+/**
+ * Check if element is a propagating element
+ */
+function isPropagatingElement(element: ElementInfo): boolean {
+  const tagName = element.tagName || element.type || '';
+  const tag = tagName.toLowerCase();
+  const role = element.role || element.attributes?.role || null;
+  
+  // For generic elements with cursor=pointer, we need to be more selective
+  // Only treat them as propagating if they don't have text content
+  // (text-containing generics are usually labels, not containers)
+  if ((tag === 'generic' || element.type === 'generic') && 
+      element.attributes?.['cursor'] === 'pointer') {
+    // If element has direct text content, it's likely a label, not a container
+    if (element.text && element.text.trim()) {
+      return false;
+    }
+    // If no text, it might be a container
+    return true;
+  }
+  
+  for (const pattern of PROPAGATING_ELEMENTS) {
+    if (pattern.tag === tag) {
+      if (pattern.role === null || pattern.role === role) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Check if child bounds are contained within parent bounds
+ */
+function isContained(
+  childBounds: { x: number; y: number; width: number; height: number },
+  parentBounds: { x: number; y: number; width: number; height: number },
+  threshold: number
+): boolean {
+  // Calculate intersection
+  const xOverlap = Math.max(0, 
+    Math.min(childBounds.x + childBounds.width, parentBounds.x + parentBounds.width) - 
+    Math.max(childBounds.x, parentBounds.x)
+  );
+  const yOverlap = Math.max(0,
+    Math.min(childBounds.y + childBounds.height, parentBounds.y + parentBounds.height) - 
+    Math.max(childBounds.y, parentBounds.y)
+  );
+  
+  const intersectionArea = xOverlap * yOverlap;
+  const childArea = childBounds.width * childBounds.height;
+  
+  if (childArea === 0) return false;
+  
+  return (intersectionArea / childArea) >= threshold;
+}
+
+/**
+ * Check if child element should be filtered out
+ */
+function shouldFilterChild(childEl: ElementInfo, parentEl: ElementInfo): boolean {
+  // Never filter if parent is not a propagating element
+  if (!isPropagatingElement(parentEl)) {
+    return false;
+  }
+  
+  // Never filter if elements don't have coordinates
+  if (!childEl.coordinates || !parentEl.coordinates) {
+    return false;
+  }
+  
+  // Check containment
+  if (!isContained(childEl.coordinates, parentEl.coordinates, CONTAINMENT_THRESHOLD)) {
+    return false;
+  }
+  
+  const childTag = (childEl.tagName || childEl.type || '').toLowerCase();
+  const childRole = childEl.role || childEl.attributes?.role || null;
+  
+  // Exception rules - never filter these:
+  
+  // 1. Form elements (need individual interaction)
+  if (['input', 'select', 'textarea', 'label'].includes(childTag)) {
+    return false;
+  }
+  
+  // 2. Child is also a propagating element (might have stopPropagation)
+  if (isPropagatingElement(childEl)) {
+    return false;
+  }
+  
+  // 3. Has onclick handler
+  if (childEl.attributes?.onclick) {
+    return false;
+  }
+  
+  // 4. Has meaningful aria-label
+  if (childEl.attributes?.['aria-label']?.trim()) {
+    return false;
+  }
+  
+  // 5. Has interactive role
+  if (['button', 'link', 'checkbox', 'radio', 'tab', 'menuitem'].includes(childRole || '')) {
+    return false;
+  }
+  
+  // Default: filter this child
+  return true;
+}
+
+/**
+ * Filter clickable elements based on parent-child relationships
+ * @param elements - Map of all elements with their info
+ * @param clickableRefs - Set of refs that are clickable
+ * @returns Filtered set of element refs and debug info
+ */
+export function filterParentChildElements(
+  elements: Record<string, ElementInfo>,
+  clickableRefs: Set<string>
+): {
+  filteredElements: Set<string>;
+  debugInfo: any[];
+} {
+  const elementRefs = Array.from(clickableRefs);
+  const filteredElements = new Set<string>(elementRefs);
+  const debugInfo: any[] = [];
+  
+  console.log(`[Parent-Child Filter] Analyzing ${elementRefs.length} clickable elements`);
+  
+  // Check each pair of elements for parent-child filtering
+  for (let i = 0; i < elementRefs.length; i++) {
+    const parentRef = elementRefs[i];
+    const parentEl = elements[parentRef];
+    
+    if (!parentEl?.coordinates) continue;
+    
+    const isParentPropagating = isPropagatingElement(parentEl);
+    
+    for (let j = 0; j < elementRefs.length; j++) {
+      if (i === j) continue;
+      
+      const childRef = elementRefs[j];
+      const childEl = elements[childRef];
+      
+      if (!childEl?.coordinates) continue;
+      
+      // Debug parent-child relationships when enabled
+      const DEBUG_PARENT_CHILD = process.env.DEBUG_PARENT_CHILD === 'true';
+      if (DEBUG_PARENT_CHILD) {
+        const shouldFilter = shouldFilterChild(childEl, parentEl);
+        console.log(`\n[Debug] Checking ${parentRef} -> ${childRef}:`);
+        console.log(`Parent:`, {
+          ref: parentRef, 
+          type: parentEl.type || parentEl.tagName, 
+          role: parentEl.role,
+          coords: parentEl.coordinates,
+          isPropagating: isParentPropagating
+        });
+        console.log(`Child:`, {
+          ref: childRef, 
+          type: childEl.type || childEl.tagName, 
+          role: childEl.role,
+          coords: childEl.coordinates
+        });
+        console.log(`Should filter? ${shouldFilter}`);
+      }
+      
+      if (shouldFilterChild(childEl, parentEl)) {
+        filteredElements.delete(childRef);
+        
+        debugInfo.push({
+          type: 'filtered',
+          childRef,
+          parentRef,
+          reason: 'Contained within propagating parent',
+          parentType: parentEl.type || parentEl.tagName,
+          childType: childEl.type || childEl.tagName,
+          parentRole: parentEl.role,
+          childRole: childEl.role,
+          containment: isContained(childEl.coordinates, parentEl.coordinates, CONTAINMENT_THRESHOLD)
+        });
+      }
+    }
+  }
+  
+  const filteredCount = elementRefs.length - filteredElements.size;
+  console.log(`[Parent-Child Filter] Filtered out ${filteredCount} child elements`);
+  
+  return {
+    filteredElements,
+    debugInfo
+  };
+}