camel-ai 0.2.75a5__py3-none-any.whl → 0.2.76a0__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/src/parent-child-filter.ts

@@ -0,0 +1,226 @@
+/**
+ * Parent-child filtering logic for SOM-labels
+ * Filters out child elements that are contained within propagating parent elements
+ */
+
+export interface ElementInfo {
+  ref: string;
+  coordinates?: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+  };
+  role?: string;
+  type?: string;
+  tagName?: string;
+  attributes?: Record<string, any>;
+  text?: string;
+}
+
+// Elements that propagate bounds to their children
+const PROPAGATING_ELEMENTS = [
+  { tag: 'a', role: null },
+  { tag: 'button', role: null },
+  { tag: 'div', role: 'button' },
+  { tag: 'div', role: 'combobox' },
+  { tag: 'span', role: 'button' },
+  { tag: 'span', role: 'combobox' },
+  { tag: 'input', role: 'combobox' },
+];
+
+const CONTAINMENT_THRESHOLD = 0.99; // 99% containment required
+
+/**
+ * Check if element is a propagating element
+ */
+function isPropagatingElement(element: ElementInfo): boolean {
+  const tagName = element.tagName || element.type || '';
+  const tag = tagName.toLowerCase();
+  const role = element.role || element.attributes?.role || null;
+  
+  // For generic elements with cursor=pointer, we need to be more selective
+  // Only treat them as propagating if they don't have text content
+  // (text-containing generics are usually labels, not containers)
+  if ((tag === 'generic' || element.type === 'generic') && 
+      element.attributes?.['cursor'] === 'pointer') {
+    // If element has direct text content, it's likely a label, not a container
+    if (element.text && element.text.trim()) {
+      return false;
+    }
+    // If no text, it might be a container
+    return true;
+  }
+  
+  for (const pattern of PROPAGATING_ELEMENTS) {
+    if (pattern.tag === tag) {
+      if (pattern.role === null || pattern.role === role) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Check if child bounds are contained within parent bounds
+ */
+function isContained(
+  childBounds: { x: number; y: number; width: number; height: number },
+  parentBounds: { x: number; y: number; width: number; height: number },
+  threshold: number
+): boolean {
+  // Calculate intersection
+  const xOverlap = Math.max(0, 
+    Math.min(childBounds.x + childBounds.width, parentBounds.x + parentBounds.width) - 
+    Math.max(childBounds.x, parentBounds.x)
+  );
+  const yOverlap = Math.max(0,
+    Math.min(childBounds.y + childBounds.height, parentBounds.y + parentBounds.height) - 
+    Math.max(childBounds.y, parentBounds.y)
+  );
+  
+  const intersectionArea = xOverlap * yOverlap;
+  const childArea = childBounds.width * childBounds.height;
+  
+  if (childArea === 0) return false;
+  
+  return (intersectionArea / childArea) >= threshold;
+}
+
+/**
+ * Check if child element should be filtered out
+ */
+function shouldFilterChild(childEl: ElementInfo, parentEl: ElementInfo): boolean {
+  // Never filter if parent is not a propagating element
+  if (!isPropagatingElement(parentEl)) {
+    return false;
+  }
+  
+  // Never filter if elements don't have coordinates
+  if (!childEl.coordinates || !parentEl.coordinates) {
+    return false;
+  }
+  
+  // Check containment
+  if (!isContained(childEl.coordinates, parentEl.coordinates, CONTAINMENT_THRESHOLD)) {
+    return false;
+  }
+  
+  const childTag = (childEl.tagName || childEl.type || '').toLowerCase();
+  const childRole = childEl.role || childEl.attributes?.role || null;
+  
+  // Exception rules - never filter these:
+  
+  // 1. Form elements (need individual interaction)
+  if (['input', 'select', 'textarea', 'label'].includes(childTag)) {
+    return false;
+  }
+  
+  // 2. Child is also a propagating element (might have stopPropagation)
+  if (isPropagatingElement(childEl)) {
+    return false;
+  }
+  
+  // 3. Has onclick handler
+  if (childEl.attributes?.onclick) {
+    return false;
+  }
+  
+  // 4. Has meaningful aria-label
+  if (childEl.attributes?.['aria-label']?.trim()) {
+    return false;
+  }
+  
+  // 5. Has interactive role
+  if (['button', 'link', 'checkbox', 'radio', 'tab', 'menuitem'].includes(childRole || '')) {
+    return false;
+  }
+  
+  // Default: filter this child
+  return true;
+}
+
+/**
+ * Filter clickable elements based on parent-child relationships
+ * @param elements - Map of all elements with their info
+ * @param clickableRefs - Set of refs that are clickable
+ * @returns Filtered set of element refs and debug info
+ */
+export function filterParentChildElements(
+  elements: Record<string, ElementInfo>,
+  clickableRefs: Set<string>
+): {
+  filteredElements: Set<string>;
+  debugInfo: any[];
+} {
+  const elementRefs = Array.from(clickableRefs);
+  const filteredElements = new Set<string>(elementRefs);
+  const debugInfo: any[] = [];
+  
+  console.log(`[Parent-Child Filter] Analyzing ${elementRefs.length} clickable elements`);
+  
+  // Check each pair of elements for parent-child filtering
+  for (let i = 0; i < elementRefs.length; i++) {
+    const parentRef = elementRefs[i];
+    const parentEl = elements[parentRef];
+    
+    if (!parentEl?.coordinates) continue;
+    
+    const isParentPropagating = isPropagatingElement(parentEl);
+    
+    for (let j = 0; j < elementRefs.length; j++) {
+      if (i === j) continue;
+      
+      const childRef = elementRefs[j];
+      const childEl = elements[childRef];
+      
+      if (!childEl?.coordinates) continue;
+      
+      // Debug parent-child relationships when enabled
+      const DEBUG_PARENT_CHILD = process.env.DEBUG_PARENT_CHILD === 'true';
+      if (DEBUG_PARENT_CHILD) {
+        const shouldFilter = shouldFilterChild(childEl, parentEl);
+        console.log(`\n[Debug] Checking ${parentRef} -> ${childRef}:`);
+        console.log(`Parent:`, {
+          ref: parentRef, 
+          type: parentEl.type || parentEl.tagName, 
+          role: parentEl.role,
+          coords: parentEl.coordinates,
+          isPropagating: isParentPropagating
+        });
+        console.log(`Child:`, {
+          ref: childRef, 
+          type: childEl.type || childEl.tagName, 
+          role: childEl.role,
+          coords: childEl.coordinates
+        });
+        console.log(`Should filter? ${shouldFilter}`);
+      }
+      
+      if (shouldFilterChild(childEl, parentEl)) {
+        filteredElements.delete(childRef);
+        
+        debugInfo.push({
+          type: 'filtered',
+          childRef,
+          parentRef,
+          reason: 'Contained within propagating parent',
+          parentType: parentEl.type || parentEl.tagName,
+          childType: childEl.type || childEl.tagName,
+          parentRole: parentEl.role,
+          childRole: childEl.role,
+          containment: isContained(childEl.coordinates, parentEl.coordinates, CONTAINMENT_THRESHOLD)
+        });
+      }
+    }
+  }
+  
+  const filteredCount = elementRefs.length - filteredElements.size;
+  console.log(`[Parent-Child Filter] Filtered out ${filteredCount} child elements`);
+  
+  return {
+    filteredElements,
+    debugInfo
+  };
+}

camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts

@@ -0,0 +1,231 @@
+/**
+ * Parse snapshot text to extract parent-child relationships
+ */
+
+export interface SnapshotNode {
+  ref: string;
+  type: string;
+  text?: string;
+  attributes: Record<string, string>;
+  children: string[]; // refs of child elements
+  parent?: string; // ref of parent element
+}
+
+export function parseSnapshotHierarchy(snapshotText: string): Map<string, SnapshotNode> {
+  const nodes = new Map<string, SnapshotNode>();
+  const lines = snapshotText.split('\n');
+  
+  // Stack to track parent elements at each indentation level
+  const parentStack: { ref: string; indent: number }[] = [];
+  
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    
+    // Calculate indentation
+    const indent = line.length - line.trimStart().length;
+    
+    // Extract element info using regex
+    // Support both lines with : (have children) and without : (leaf nodes)
+    // Also support quoted lines like - 'button "text" [ref=...]'
+    // Also support escaped quotes in text
+    // Also support attributes before [ref=...]
+    // Extract type and optional label (before any [..] blocks)
+    const headerMatch = line.match(/^\s*(?:-\s*)?'?([a-z0-9_-]+)(?:\s+"((?:[^"\\]|\\.)*)")?/i);
+    if (!headerMatch) continue;
+    const [, typeRaw, label] = headerMatch;
+    const type = (typeRaw || 'unknown');
+
+    // Extract mandatory ref
+    const refMatch = line.match(/\[ref=([^\]]+)\]/i);
+    if (!refMatch) continue;
+    const ref = refMatch[1];
+
+    // Parse all bracketed attributes except the [ref=...] block
+    const attrs: Record<string, string> = {};
+    for (const block of line.matchAll(/\[([^\]]+)\]/g)) {
+      const content = block[1];
+      if (/^ref=/i.test(content)) continue;
+      const attrMatches = content.matchAll(/([a-z0-9_-]+)(?:=(?:"([^"]*)"|'([^']*)'|([^\s\]]+)))?/gi);
+      for (const m of attrMatches) {
+        const key = m[1].toLowerCase();
+        const val = m[2] ?? m[3] ?? m[4] ?? 'true';
+        attrs[key] = val;
+      }
+    }
+    
+    // Update parent stack based on indentation
+    while (parentStack.length > 0 && parentStack[parentStack.length - 1].indent >= indent) {
+      parentStack.pop();
+    }
+    
+    // Create node
+    const node: SnapshotNode = {
+      ref,
+      type: type.toLowerCase(),
+      text: label || '',
+      attributes: attrs,
+      children: [],
+      parent: parentStack.length > 0 ? parentStack[parentStack.length - 1].ref : undefined
+    };
+    
+    // Add to parent's children if has parent
+    if (node.parent && nodes.has(node.parent)) {
+      nodes.get(node.parent)!.children.push(ref);
+    }
+    
+    // Add to nodes map
+    nodes.set(ref, node);
+    
+    // Add to parent stack
+    parentStack.push({ ref, indent });
+  }
+  
+  return nodes;
+}
+
+/**
+ * Find clickable elements that should be filtered based on DOM hierarchy
+ */
+export function filterClickableByHierarchy(
+  snapshotText: string,
+  clickableElements: Set<string>
+): Set<string> {
+  const hierarchy = parseSnapshotHierarchy(snapshotText);
+  const filtered = new Set<string>(clickableElements);
+  const debugInfo: any[] = [];
+  
+  // Debug clickable elements when enabled
+  const DEBUG_SNAPSHOT_PARSER = process.env.DEBUG_SNAPSHOT_PARSER === 'true';
+  if (DEBUG_SNAPSHOT_PARSER) {
+    clickableElements.forEach(ref => {
+      const node = hierarchy.get(ref);
+      if (node) {
+        console.log(`[Debug] ${ref} type: ${node.type}, parent: ${node.parent || 'none'}`);
+      } else {
+        console.log(`[Debug] ${ref} NOT FOUND in hierarchy!`);
+      }
+    });
+  }
+  
+  // First pass: identify parent-child relationships where both are clickable
+  const parentChildPairs: Array<{parent: string, child: string, parentType: string, childType: string}> = [];
+  
+  for (const childRef of clickableElements) {
+    const childNode = hierarchy.get(childRef);
+    if (!childNode || !childNode.parent) continue;
+    
+    const parentRef = childNode.parent;
+    if (clickableElements.has(parentRef)) {
+      const parentNode = hierarchy.get(parentRef);
+      if (parentNode) {
+        parentChildPairs.push({
+          parent: parentRef,
+          child: childRef,
+          parentType: parentNode.type.toLowerCase(),
+          childType: childNode.type.toLowerCase()
+        });
+        
+        // Debug specific pairs
+        if ((parentRef === 'e296' && childRef === 'e297') ||
+            (parentRef === 'e361' && childRef === 'e363') ||
+            (parentRef === 'e371' && childRef === 'e373') ||
+            (parentRef === 'e344' && childRef === 'e346') ||
+            (parentRef === 'e348' && childRef === 'e350')) {
+          console.log(`[Debug] Found pair: ${parentRef} (${parentNode.type}) -> ${childRef} (${childNode.type})`);
+        }
+      }
+    }
+  }
+  
+  // Decide which elements to filter based on parent-child relationships
+  for (const pair of parentChildPairs) {
+    const { parent, child, parentType, childType } = pair;
+    
+    // Rules for what to filter:
+    // 1. link > img: filter img (keep link)
+    // 2. button > generic: filter generic (keep button)  
+    // 3. generic > button: filter generic (keep button)
+    // 4. link > generic: filter generic (keep link)
+    // 5. generic > generic: filter child (keep parent)
+    // 6. generic > unknown: filter child (keep parent)
+    
+    if ((parentType === 'link' && childType === 'img') ||
+        (parentType === 'button' && childType === 'generic') ||
+        (parentType === 'link' && childType === 'generic') ||
+        (parentType === 'generic' && childType === 'generic') ||
+        (parentType === 'generic' && childType === 'unknown')) {
+      // Filter child
+      filtered.delete(child);
+      console.log(`[Hierarchy Filter] Filtered ${child} (${childType}) - keeping parent ${parent} (${parentType})`);
+    } else if (parentType === 'generic' && childType === 'button') {
+      // Special case: filter parent generic, keep child button
+      filtered.delete(parent);
+      console.log(`[Hierarchy Filter] Filtered ${parent} (${parentType}) - keeping child ${child} (${childType})`);
+    }
+  }
+  
+  // Original logic for nested hierarchies (keep for deep nesting)
+  for (const childRef of clickableElements) {
+    if (!filtered.has(childRef)) continue; // Already filtered
+    
+    const childNode = hierarchy.get(childRef);
+    if (!childNode || !childNode.parent) continue;
+    
+    // Check if any ancestor is a propagating element
+    let currentParent: string | undefined = childNode.parent;
+    while (currentParent) {
+      if (clickableElements.has(currentParent) && filtered.has(currentParent)) {
+        const parentNode = hierarchy.get(currentParent);
+        if (parentNode) {
+          // Check if parent is a propagating element
+          const parentType = parentNode.type.toLowerCase();
+          const isPropagating = ['button', 'link', 'a'].includes(parentType) ||
+            (parentType === 'generic' && parentNode.attributes.cursor === 'pointer' && !parentNode.text);
+          
+          if (isPropagating) {
+            // Filter child elements that should be contained within propagating parents
+            const childType = childNode.type.toLowerCase();
+            
+            // Filter these types of children:
+            // 1. Generic elements with cursor=pointer
+            // 2. Images within links/buttons
+            // 3. Text elements (span, generic without specific role)
+            const shouldFilter = 
+              (childType === 'generic' && childNode.attributes.cursor === 'pointer') ||
+              childType === 'img' ||
+              childType === 'span' ||
+              (childType === 'generic' && !childNode.attributes.role);
+              
+            if (shouldFilter) {
+              filtered.delete(childRef);
+              console.log(`[Hierarchy Filter] Filtered ${childRef} (${childType}) contained in ${currentParent} (${parentType})`);
+              break;
+            }
+          }
+        }
+      }
+      // Move up the hierarchy
+      const nextParent = hierarchy.get(currentParent);
+      currentParent = nextParent?.parent;
+    }
+  }
+  
+  // Additional pass: if a generic parent contains only one button child, filter the parent
+  for (const ref of Array.from(filtered)) {
+    const node = hierarchy.get(ref);
+    if (!node || node.type.toLowerCase() !== 'generic') continue;
+    
+    // Check if this generic has exactly one clickable child that's a button
+    const clickableChildren = node.children.filter(childRef => 
+      filtered.has(childRef) && hierarchy.get(childRef)?.type.toLowerCase() === 'button'
+    );
+    
+    if (clickableChildren.length === 1) {
+      // This generic wraps a single button - filter it out
+      filtered.delete(ref);
+      console.log(`[Hierarchy Filter] Filtered ${ref} (generic wrapper around button ${clickableChildren[0]})`);
+    }
+  }
+  
+  return filtered;
+}