agentcrew-ai 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

AgentCrew/modules/browser_automation/js/draw_element_boxes.js

@@ -0,0 +1,200 @@
+/**
+ * Draw colored rectangle boxes with UUID labels over elements
+ *
+ * @param {Object} uuidXpathMap - Map of UUID to XPath selector
+ * @returns {Object} Result object with success status and message
+ */
+function drawElementBoxes(uuidXpathMap) {
+  try {
+    const existingContainer = document.getElementById(
+      "agentcrew-element-overlay-container",
+    );
+    if (existingContainer) {
+      existingContainer.remove();
+    }
+
+    const svgNS = "http://www.w3.org/2000/svg";
+    const container = document.createElementNS(svgNS, "svg");
+    container.setAttribute("id", "agentcrew-element-overlay-container");
+    container.style.position = "fixed";
+    container.style.top = "0";
+    container.style.left = "0";
+    container.style.width = "100%";
+    container.style.height = "100%";
+    container.style.pointerEvents = "none";
+    container.style.zIndex = "2147483647";
+
+    const colors = [
+      "#FF6B6B",
+      "#4ECDC4",
+      "#45B7D1",
+      "#FFA07A",
+      "#98D8C8",
+      "#F7DC6F",
+      "#BB8FCE",
+      "#85C1E2",
+      "#F8B739",
+      "#52B788",
+    ];
+
+    const labelPadding = 8;
+    const fontSize = 14;
+    const labelHeight = 24;
+    const occupiedSpaces = [];
+
+    function checkOverlap(newLabel, existingLabels) {
+      for (const existing of existingLabels) {
+        if (
+          !(
+            newLabel.right < existing.left ||
+            newLabel.left > existing.right ||
+            newLabel.bottom < existing.top ||
+            newLabel.top > existing.bottom
+          )
+        ) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    function findBestLabelPosition(rect, labelWidth, existingLabels) {
+      const positions = [
+        { x: rect.left, y: rect.top - labelHeight - 4, priority: 1 },
+        { x: rect.left, y: rect.bottom + 4, priority: 2 },
+        {
+          x: rect.right - labelWidth,
+          y: rect.top - labelHeight - 4,
+          priority: 3,
+        },
+        { x: rect.right - labelWidth, y: rect.bottom + 4, priority: 4 },
+        {
+          x: rect.left + rect.width / 2 - labelWidth / 2,
+          y: rect.top - labelHeight - 4,
+          priority: 5,
+        },
+        { x: rect.left - labelWidth - 4, y: rect.top, priority: 6 },
+        { x: rect.right + 4, y: rect.top, priority: 7 },
+        {
+          x: rect.left,
+          y: rect.top + rect.height / 2 - labelHeight / 2,
+          priority: 8,
+        },
+      ];
+
+      positions.sort((a, b) => a.priority - b.priority);
+
+      for (const pos of positions) {
+        const labelBounds = {
+          left: pos.x,
+          right: pos.x + labelWidth,
+          top: pos.y,
+          bottom: pos.y + labelHeight,
+        };
+
+        if (labelBounds.top >= 0 && labelBounds.left >= 0) {
+          if (!checkOverlap(labelBounds, existingLabels)) {
+            return { x: pos.x, y: pos.y };
+          }
+        }
+      }
+
+      return { x: rect.left, y: Math.max(0, rect.top - labelHeight - 4) };
+    }
+
+    let colorIndex = 0;
+    let drawnCount = 0;
+
+    for (const [uuid, xpath] of Object.entries(uuidXpathMap)) {
+      try {
+        const result = document.evaluate(
+          xpath,
+          document,
+          null,
+          XPathResult.FIRST_ORDERED_NODE_TYPE,
+          null,
+        );
+        const element = result.singleNodeValue;
+
+        if (!element) {
+          continue;
+        }
+
+        const rect = element.getBoundingClientRect();
+
+        if (rect.width === 0 || rect.height === 0) {
+          continue;
+        }
+
+        const color = colors[colorIndex % colors.length];
+        colorIndex++;
+
+        const group = document.createElementNS(svgNS, "g");
+
+        const box = document.createElementNS(svgNS, "rect");
+        box.setAttribute("x", rect.left);
+        box.setAttribute("y", rect.top);
+        box.setAttribute("width", rect.width);
+        box.setAttribute("height", rect.height);
+        box.setAttribute("fill", "none");
+        box.setAttribute("stroke", color);
+        box.setAttribute("stroke-width", "3");
+        box.setAttribute("stroke-dasharray", "5,5");
+        box.setAttribute("rx", "4");
+
+        const labelWidth = uuid.length * (fontSize * 0.6) + labelPadding * 2;
+        const labelPos = findBestLabelPosition(
+          rect,
+          labelWidth,
+          occupiedSpaces,
+        );
+
+        const labelBg = document.createElementNS(svgNS, "rect");
+        labelBg.setAttribute("x", labelPos.x);
+        labelBg.setAttribute("y", labelPos.y);
+        labelBg.setAttribute("width", labelWidth);
+        labelBg.setAttribute("height", labelHeight);
+        labelBg.setAttribute("fill", color);
+        labelBg.setAttribute("rx", "4");
+
+        const label = document.createElementNS(svgNS, "text");
+        label.setAttribute("x", labelPos.x + labelPadding);
+        label.setAttribute("y", labelPos.y + 17);
+        label.setAttribute("fill", "#FFFFFF");
+        label.setAttribute("font-family", "monospace");
+        label.setAttribute("font-size", fontSize);
+        label.setAttribute("font-weight", "bold");
+        label.textContent = uuid;
+
+        occupiedSpaces.push({
+          left: labelPos.x,
+          right: labelPos.x + labelWidth,
+          top: labelPos.y,
+          bottom: labelPos.y + labelHeight,
+        });
+
+        group.appendChild(box);
+        group.appendChild(labelBg);
+        group.appendChild(label);
+        container.appendChild(group);
+
+        drawnCount++;
+      } catch (err) {
+        console.warn(`Failed to draw box for UUID ${uuid}:`, err);
+      }
+    }
+
+    document.body.appendChild(container);
+
+    return {
+      success: true,
+      message: `Successfully drew ${drawnCount} element boxes`,
+      count: drawnCount,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error.message,
+    };
+  }
+}

AgentCrew/modules/browser_automation/js/extract_clickable_elements.js

@@ -10,6 +10,20 @@
   const seenHrefs = new Set();
   const seenElements = new Set();
 
+  function isInViewport(rect) {
+    const viewportWidth =
+      window.innerWidth || document.documentElement.clientWidth;
+    const viewportHeight =
+      window.innerHeight || document.documentElement.clientHeight;
+
+    return (
+      rect.top < viewportHeight &&
+      rect.bottom > 0 &&
+      rect.left < viewportWidth &&
+      rect.right > 0
+    );
+  }
+
   // Utility function to check if element is truly visible (including parent chain)
   function isElementVisible(element) {
     if (!element || !element.nodeType === 1) {
@@ -21,27 +35,24 @@
       return false;
     }
 
-    // Walk up the parent chain checking visibility
-    let currentElement = element;
+    if (
+      element.tagName.toLowerCase() === "input" &&
+      !element.checkVisibility()
+    ) {
+      //go up to parent if input is not visible
+      return element.parentElement.checkVisibility();
+    }
 
-    if (currentElement.clientWidth <= 1 && currentElement.clientHeight <= 1) {
+    if (!element.checkVisibility()) {
       return false;
     }
 
-    while (
-      currentElement &&
-      currentElement !== document.body &&
-      currentElement !== document.documentElement
-    ) {
-      const style = window.getComputedStyle(currentElement);
-
-      // Check if current element is hidden
-      if (style.display === "none" || style.visibility === "hidden") {
-        return false;
-      }
-
-      // Move to parent element
-      currentElement = currentElement.parentElement;
+    bounding_box = element.getBoundingClientRect();
+    if (!isInViewport(bounding_box)) {
+      return false;
+    }
+    if (bounding_box.width <= 1 && bounding_box.height <= 1) {
+      return false;
     }
 
     return true;
@@ -80,6 +91,8 @@
     'input[type="button"]',
     'input[type="submit"]',
     'input[type="reset"]',
+    'input[type="checkbox"]',
+    'input[type="radio"]',
     "[onclick]", // Elements with onclick handlers
     '[role="button"]', // ARIA buttons
     "[tabindex]", // Focusable elements
@@ -101,7 +114,14 @@
       const href = element.href || element.getAttribute("href") || "";
 
       // Generate XPath
-      const xpath = getXPath(element);
+      let xpath = getXPath(element);
+
+      if (
+        element.tagName.toLowerCase() === "input" &&
+        !element.checkVisibility()
+      ) {
+        xpath = getXPath(element.parentElement);
+      }
 
       // Get display text
       let displayText = element.getAttribute("aria-label");
@@ -119,7 +139,7 @@
             altTexts.push(label);
           }
         });
-        if (altTexts.length > 0 && !displayText) {
+        if (altTexts.length > 0 && displayText) {
           displayText = altTexts.join(", ");
         }
       }
@@ -133,13 +153,26 @@
         if (!displayText) {
           displayText = element.title || "";
         }
+      }
 
-        // Limit text length
-        if (displayText.length > 50) {
-          displayText = displayText.substring(0, 50) + "...";
+      let elementType = element.tagName.toLowerCase();
+      if (elementType === "input") {
+        elementType = element.type;
+        if (element.type === "checkbox" || element.type === "radio") {
+          elementType =
+            element.type + "_" + element.name + "[" + element.value + "]";
+          let parent = element.parentElement;
+          while (!displayText) {
+            displayText = parent.textContent || "";
+            parent = parent.parentElement;
+          }
         }
       }
 
+      // Limit text length
+      if (displayText.length > 50) {
+        displayText = displayText.substring(0, 50) + "...";
+      }
       // Only add if we have some meaningful content
       if (displayText || xpath) {
         // Deduplication logic
@@ -153,11 +186,10 @@
             });
           }
         } else {
-          // For elements without href, deduplicate by tagName + text combination
-          const elementKey = element.tagName.toLowerCase() + "|" + displayText;
-          if (!seenElements.has(elementKey)) {
-            seenElements.add(elementKey);
+          if (!seenElements.has(xpath)) {
+            seenElements.add(xpath);
             clickableElements.push({
+              type: elementType,
               xpath: xpath,
               text: displayText,
             });

AgentCrew/modules/browser_automation/js/extract_elements_by_text.js

@@ -8,33 +8,36 @@
 function extractElementsByText(text) {
   const elementsFound = [];
 
+  function isInViewport(rect) {
+    const viewportWidth =
+      window.innerWidth || document.documentElement.clientWidth;
+    const viewportHeight =
+      window.innerHeight || document.documentElement.clientHeight;
+
+    return (
+      rect.top < viewportHeight &&
+      rect.bottom > 0 &&
+      rect.left < viewportWidth &&
+      rect.right > 0
+    );
+  }
+
   // Utility function to check if element is truly visible (including parent chain)
   function isElementVisible(element) {
     if (!element || !element.nodeType === 1) {
       return false;
     }
 
-    // Walk up the parent chain checking visibility
-    let currentElement = element;
-
-    if (currentElement.clientWidth <= 1 && currentElement.clientHeight <= 1) {
+    if (!element.checkVisibility()) {
       return false;
     }
 
-    while (
-      currentElement &&
-      currentElement !== document.body &&
-      currentElement !== document.documentElement
-    ) {
-      const style = window.getComputedStyle(currentElement);
-
-      // Check if current element is hidden
-      if (style.display === "none" || style.visibility === "hidden") {
-        return false;
-      }
-
-      // Move to parent element
-      currentElement = currentElement.parentElement;
+    bounding_box = element.getBoundingClientRect();
+    if (!isInViewport(bounding_box)) {
+      return false;
+    }
+    if (bounding_box.width <= 1 && bounding_box.height <= 1) {
+      return false;
     }
 
     return true;
@@ -120,4 +123,3 @@ function extractElementsByText(text) {
 //     const text = '{TEXT_PLACEHOLDER}';
 //     return extractElementsByText(text);
 // })();
-

AgentCrew/modules/browser_automation/js/extract_input_elements.js

@@ -8,33 +8,36 @@
   const inputElements = [];
   const seenElements = new Set();
 
+  function isInViewport(rect) {
+    const viewportWidth =
+      window.innerWidth || document.documentElement.clientWidth;
+    const viewportHeight =
+      window.innerHeight || document.documentElement.clientHeight;
+
+    return (
+      rect.top < viewportHeight &&
+      rect.bottom > 0 &&
+      rect.left < viewportWidth &&
+      rect.right > 0
+    );
+  }
+
   // Utility function to check if element is truly visible (including parent chain)
   function isElementVisible(element) {
     if (!element || !element.nodeType === 1) {
       return false;
     }
 
-    // Walk up the parent chain checking visibility
-    let currentElement = element;
-
-    if (currentElement.clientWidth <= 1 && currentElement.clientHeight <= 1) {
+    if (!element.checkVisibility()) {
       return false;
     }
 
-    while (
-      currentElement &&
-      currentElement !== document.body &&
-      currentElement !== document.documentElement
-    ) {
-      const style = window.getComputedStyle(currentElement);
-
-      // Check if current element is hidden
-      if (style.display === "none" || style.visibility === "hidden") {
-        return false;
-      }
-
-      // Move to parent element
-      currentElement = currentElement.parentElement;
+    bounding_box = element.getBoundingClientRect();
+    if (!isInViewport(bounding_box)) {
+      return false;
+    }
+    if (bounding_box.width <= 1 && bounding_box.height <= 1) {
+      return false;
     }
 
     return true;
@@ -113,8 +116,6 @@
     'input[type="email"]',
     'input[type="password"]',
     'input[type="number"]',
-    'input[type="checkbox"]',
-    'input[type="radio"]',
     'input[type="tel"]',
     'input[type="url"]',
     'input[type="search"]',
@@ -198,9 +199,7 @@
 
       // Get current value
       let elementValue = "";
-      if (elementType === "checkbox" || elementType === "radio") {
-        elementValue = element.checked ? "checked" : "unchecked";
-      } else if (elementType === "select") {
+      if (elementType === "select") {
         elementValue = element.value || "";
         // Also show selected text for better understanding
         if (element.selectedOptions && element.selectedOptions.length > 0) {

AgentCrew/modules/browser_automation/js/filter_hidden_elements.js

@@ -0,0 +1,104 @@
+function filterHiddenElements() {
+  try {
+    function isElementHidden(element) {
+      if (element.nodeType !== Node.ELEMENT_NODE) {
+        return false;
+      }
+
+      const tagName = element.tagName.toLowerCase();
+      if (
+        tagName === "script" ||
+        tagName === "style" ||
+        tagName === "noscript"
+      ) {
+        return true;
+      }
+
+      const computedStyle = window.getComputedStyle(element);
+
+      if (
+        computedStyle.display === "none" ||
+        computedStyle.visibility === "hidden" ||
+        computedStyle.opacity === "0"
+      ) {
+        return true;
+      }
+
+      const ariaHidden = element.getAttribute("aria-hidden");
+      if (ariaHidden === "true") {
+        return true;
+      }
+
+      return false;
+    }
+
+    function removeHiddenElementsFromNode(node) {
+      if (!node || node.nodeType !== Node.ELEMENT_NODE) {
+        return;
+      }
+
+      const children = Array.from(node.children);
+
+      for (const child of children) {
+        const originalElement = document.querySelector(
+          `[ag-data-temp-id="${child.getAttribute("ag-data-temp-id")}"]`,
+        );
+
+        if (!originalElement) {
+          child.remove();
+          continue;
+        }
+
+        if (isElementHidden(originalElement)) {
+          child.remove();
+        } else {
+          removeHiddenElementsFromNode(child);
+        }
+      }
+    }
+
+    function addTempIds(node, prefix = "") {
+      if (!node || node.nodeType !== Node.ELEMENT_NODE) {
+        return;
+      }
+
+      const children = Array.from(node.children);
+      children.forEach((child, index) => {
+        const tempId = `${prefix}${index}`;
+        child.setAttribute("ag-data-temp-id", tempId);
+        addTempIds(child, `${tempId}-`);
+      });
+    }
+
+    function removeTempIds(node) {
+      if (!node || node.nodeType !== Node.ELEMENT_NODE) {
+        return;
+      }
+
+      node.removeAttribute("ag-data-temp-id");
+      Array.from(node.children).forEach((child) => removeTempIds(child));
+    }
+
+    addTempIds(document.documentElement);
+    const documentClone = document.documentElement.cloneNode(true);
+
+    removeHiddenElementsFromNode(documentClone);
+
+    removeTempIds(document.documentElement);
+    removeTempIds(documentClone);
+
+    const filteredHTML = documentClone.outerHTML;
+
+    return {
+      success: true,
+      html: filteredHTML,
+      message: "Successfully filtered hidden elements using computed styles",
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error.message,
+      stack: error.stack,
+    };
+  }
+}

AgentCrew/modules/browser_automation/js/remove_element_boxes.js

@@ -0,0 +1,29 @@
+/**
+ * Remove the overlay container with element boxes
+ * 
+ * @returns {Object} Result object with success status and message
+ */
+function removeElementBoxes() {
+  try {
+    const container = document.getElementById('agentcrew-element-overlay-container');
+    
+    if (!container) {
+      return {
+        success: true,
+        message: 'No overlay container found (already removed or never created)'
+      };
+    }
+    
+    container.remove();
+    
+    return {
+      success: true,
+      message: 'Successfully removed element boxes overlay'
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error.message
+    };
+  }
+}