lumivor 0.1.7__py3-none-any.whl

lumivor/dom/buildDomTree.js

@@ -0,0 +1,428 @@
+(
+    doHighlightElements = true
+) => {
+    let highlightIndex = 0; // Reset highlight index
+
+    function highlightElement(element, index, parentIframe = null) {
+        // Create or get highlight container
+        let container = document.getElementById('playwright-highlight-container');
+        if (!container) {
+            container = document.createElement('div');
+            container.id = 'playwright-highlight-container';
+            container.style.position = 'fixed';
+            container.style.pointerEvents = 'none';
+            container.style.top = '0';
+            container.style.left = '0';
+            container.style.width = '100%';
+            container.style.height = '100%';
+            container.style.zIndex = '2147483647'; // Maximum z-index value
+            document.documentElement.appendChild(container);
+        }
+
+        // Generate a color based on the index
+        const colors = [
+            '#FF0000', '#00FF00', '#0000FF', '#FFA500', 
+            '#800080', '#008080', '#FF69B4', '#4B0082',
+            '#FF4500', '#2E8B57', '#DC143C', '#4682B4'
+        ];
+        const colorIndex = index % colors.length;
+        const baseColor = colors[colorIndex];
+        const backgroundColor = `${baseColor}1A`; // 10% opacity version of the color
+
+        // Create highlight overlay
+        const overlay = document.createElement('div');
+        overlay.style.position = 'absolute';
+        overlay.style.border = `2px solid ${baseColor}`;
+        overlay.style.backgroundColor = backgroundColor;
+        overlay.style.pointerEvents = 'none';
+        overlay.style.boxSizing = 'border-box';
+
+        // Position overlay based on element
+        const rect = element.getBoundingClientRect();
+        let top = rect.top;
+        let left = rect.left;
+
+        // Adjust position if element is inside an iframe
+        if (parentIframe) {
+            const iframeRect = parentIframe.getBoundingClientRect();
+            top += iframeRect.top;
+            left += iframeRect.left;
+        }
+
+        overlay.style.top = `${top}px`;
+        overlay.style.left = `${left}px`;
+        overlay.style.width = `${rect.width}px`;
+        overlay.style.height = `${rect.height}px`;
+
+        // Create label
+        const label = document.createElement('div');
+        label.className = 'playwright-highlight-label';
+        label.style.position = 'absolute';
+        label.style.background = baseColor;
+        label.style.color = 'white';
+        label.style.padding = '1px 4px';
+        label.style.borderRadius = '4px';
+        label.style.fontSize = `${Math.min(12, Math.max(8, rect.height / 2))}px`; // Responsive font size
+        label.textContent = index;
+
+        // Calculate label position
+        const labelWidth = 20; // Approximate width
+        const labelHeight = 16; // Approximate height
+        
+        // Default position (top-right corner inside the box)
+        let labelTop = top + 2;
+        let labelLeft = left + rect.width - labelWidth - 2;
+
+        // Adjust if box is too small
+        if (rect.width < labelWidth + 4 || rect.height < labelHeight + 4) {
+            // Position outside the box if it's too small
+            labelTop = top - labelHeight - 2;
+            labelLeft = left + rect.width - labelWidth;
+        }
+
+        // Ensure label stays within viewport
+        if (labelTop < 0) labelTop = top + 2;
+        if (labelLeft < 0) labelLeft = left + 2;
+        if (labelLeft + labelWidth > window.innerWidth) {
+            labelLeft = left + rect.width - labelWidth - 2;
+        }
+
+        label.style.top = `${labelTop}px`;
+        label.style.left = `${labelLeft}px`;
+
+        // Add to container
+        container.appendChild(overlay);
+        container.appendChild(label);
+
+        // Store reference for cleanup
+        element.setAttribute('browser-user-highlight-id', `playwright-highlight-${index}`);
+
+        return index + 1;
+    }
+
+
+    // Helper function to generate XPath as a tree
+    function getXPathTree(element, stopAtBoundary = true) {
+        const segments = [];
+        let currentElement = element;
+
+        while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
+            // Stop if we hit a shadow root or iframe
+            if (stopAtBoundary && (currentElement.parentNode instanceof ShadowRoot || currentElement.parentNode instanceof HTMLIFrameElement)) {
+                break;
+            }
+
+            let index = 0;
+            let sibling = currentElement.previousSibling;
+            while (sibling) {
+                if (sibling.nodeType === Node.ELEMENT_NODE &&
+                    sibling.nodeName === currentElement.nodeName) {
+                    index++;
+                }
+                sibling = sibling.previousSibling;
+            }
+
+            const tagName = currentElement.nodeName.toLowerCase();
+            const xpathIndex = index > 0 ? `[${index + 1}]` : '';
+            segments.unshift(`${tagName}${xpathIndex}`);
+
+            currentElement = currentElement.parentNode;
+        }
+
+        return segments.join('/');
+    }
+
+    // Helper function to check if element is accepted
+    function isElementAccepted(element) {
+        const leafElementDenyList = new Set(['svg', 'script', 'style', 'link', 'meta']);
+        return !leafElementDenyList.has(element.tagName.toLowerCase());
+    }
+
+    // Helper function to check if element is interactive
+    function isInteractiveElement(element) {
+        // Base interactive elements and roles
+        const interactiveElements = new Set([
+            'a', 'button', 'details', 'embed', 'input', 'label',
+            'menu', 'menuitem', 'object', 'select', 'textarea', 'summary'
+        ]);
+
+        const interactiveRoles = new Set([
+            'button', 'menu', 'menuitem', 'link', 'checkbox', 'radio',
+            'slider', 'tab', 'tabpanel', 'textbox', 'combobox', 'grid',
+            'listbox', 'option', 'progressbar', 'scrollbar', 'searchbox',
+            'switch', 'tree', 'treeitem', 'spinbutton', 'tooltip', 'a-button-inner', 'a-dropdown-button', 'click',
+            'menuitemcheckbox', 'menuitemradio', 'a-button-text', 'button-text', 'button-icon', 'button-icon-only', 'button-text-icon-only', 'dropdown', 'combobox' 
+        ]);
+
+        const tagName = element.tagName.toLowerCase();
+        const role = element.getAttribute('role');
+        const ariaRole = element.getAttribute('aria-role');
+        const tabIndex = element.getAttribute('tabindex');
+
+        // Basic role/attribute checks
+        const hasInteractiveRole = interactiveElements.has(tagName) ||
+            interactiveRoles.has(role) ||
+            interactiveRoles.has(ariaRole) ||
+            (tabIndex !== null && tabIndex !== '-1') ||
+            element.getAttribute('data-action') === 'a-dropdown-select' ||
+            element.getAttribute('data-action') === 'a-dropdown-button';
+
+        if (hasInteractiveRole) return true;
+
+        // Get computed style
+        const style = window.getComputedStyle(element);
+
+        // Check if element has click-like styling
+        // const hasClickStyling = style.cursor === 'pointer' ||
+        //     element.style.cursor === 'pointer' ||
+        //     style.pointerEvents !== 'none';
+
+        // Check for event listeners
+        const hasClickHandler = element.onclick !== null ||
+            element.getAttribute('onclick') !== null ||
+            element.hasAttribute('ng-click') ||
+            element.hasAttribute('@click') ||
+            element.hasAttribute('v-on:click');
+
+        // Helper function to safely get event listeners
+        function getEventListeners(el) {
+            try {
+                // Try to get listeners using Chrome DevTools API
+                return window.getEventListeners?.(el) || {};
+            } catch (e) {
+                // Fallback: check for common event properties
+                const listeners = {};
+
+                // List of common event types to check
+                const eventTypes = [
+                    'click', 'mousedown', 'mouseup',
+                    'touchstart', 'touchend',
+                    'keydown', 'keyup', 'focus', 'blur'
+                ];
+
+                for (const type of eventTypes) {
+                    const handler = el[`on${type}`];
+                    if (handler) {
+                        listeners[type] = [{
+                            listener: handler,
+                            useCapture: false
+                        }];
+                    }
+                }
+
+                return listeners;
+            }
+        }
+
+        // Check for click-related events on the element itself
+        const listeners = getEventListeners(element);
+        const hasClickListeners = listeners && (
+            listeners.click?.length > 0 ||
+            listeners.mousedown?.length > 0 ||
+            listeners.mouseup?.length > 0 ||
+            listeners.touchstart?.length > 0 ||
+            listeners.touchend?.length > 0
+        );
+
+        // Check for ARIA properties that suggest interactivity
+        const hasAriaProps = element.hasAttribute('aria-expanded') ||
+            element.hasAttribute('aria-pressed') ||
+            element.hasAttribute('aria-selected') ||
+            element.hasAttribute('aria-checked');
+
+        // Check for form-related functionality
+        const isFormRelated = element.form !== undefined ||
+            element.hasAttribute('contenteditable') ||
+            style.userSelect !== 'none';
+
+        // Check if element is draggable
+        const isDraggable = element.draggable ||
+            element.getAttribute('draggable') === 'true';
+
+        return hasAriaProps ||
+            // hasClickStyling ||
+            hasClickHandler ||
+            hasClickListeners ||
+            // isFormRelated ||
+            isDraggable;
+
+    }
+
+    // Helper function to check if element is visible
+    function isElementVisible(element) {
+        const style = window.getComputedStyle(element);
+        return element.offsetWidth > 0 &&
+            element.offsetHeight > 0 &&
+            style.visibility !== 'hidden' &&
+            style.display !== 'none';
+    }
+
+    // Helper function to check if element is the top element at its position
+    function isTopElement(element) {
+        // Find the correct document context and root element
+        let doc = element.ownerDocument;
+
+        // If we're in an iframe, elements are considered top by default
+        if (doc !== window.document) {
+            return true;
+        }
+
+        // For shadow DOM, we need to check within its own root context
+        const shadowRoot = element.getRootNode();
+        if (shadowRoot instanceof ShadowRoot) {
+            const rect = element.getBoundingClientRect();
+            const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
+
+            try {
+                // Use shadow root's elementFromPoint to check within shadow DOM context
+                const topEl = shadowRoot.elementFromPoint(point.x, point.y);
+                if (!topEl) return false;
+
+                // Check if the element or any of its parents match our target element
+                let current = topEl;
+                while (current && current !== shadowRoot) {
+                    if (current === element) return true;
+                    current = current.parentElement;
+                }
+                return false;
+            } catch (e) {
+                return true; // If we can't determine, consider it visible
+            }
+        }
+
+        // Regular DOM elements
+        const rect = element.getBoundingClientRect();
+        const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
+
+        try {
+            const topEl = document.elementFromPoint(point.x, point.y);
+            if (!topEl) return false;
+
+            let current = topEl;
+            while (current && current !== document.documentElement) {
+                if (current === element) return true;
+                current = current.parentElement;
+            }
+            return false;
+        } catch (e) {
+            return true;
+        }
+    }
+
+    // Helper function to check if text node is visible
+    function isTextNodeVisible(textNode) {
+        const range = document.createRange();
+        range.selectNodeContents(textNode);
+        const rect = range.getBoundingClientRect();
+
+        return rect.width !== 0 &&
+            rect.height !== 0 &&
+            rect.top >= 0 &&
+            rect.top <= window.innerHeight &&
+            textNode.parentElement?.checkVisibility({
+                checkOpacity: true,
+                checkVisibilityCSS: true
+            });
+    }
+
+
+    // Function to traverse the DOM and create nested JSON
+    function buildDomTree(node, parentIframe = null) {
+        if (!node) return null;
+
+        // Special case for text nodes
+        if (node.nodeType === Node.TEXT_NODE) {
+            const textContent = node.textContent.trim();
+            if (textContent && isTextNodeVisible(node)) {
+                return {
+                    type: "TEXT_NODE",
+                    text: textContent,
+                    isVisible: true,
+                };
+            }
+            return null;
+        }
+
+        // Check if element is accepted
+        if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) {
+            return null;
+        }
+
+        const nodeData = {
+            tagName: node.tagName ? node.tagName.toLowerCase() : null,
+            attributes: {},
+            xpath: node.nodeType === Node.ELEMENT_NODE ? getXPathTree(node, true) : null,
+            children: [],
+        };
+
+        // Copy all attributes if the node is an element
+        if (node.nodeType === Node.ELEMENT_NODE && node.attributes) {
+            // Use getAttributeNames() instead of directly iterating attributes
+            const attributeNames = node.getAttributeNames?.() || [];
+            for (const name of attributeNames) {
+                nodeData.attributes[name] = node.getAttribute(name);
+            }
+        }
+
+        if (node.nodeType === Node.ELEMENT_NODE) {
+            const isInteractive = isInteractiveElement(node);
+            const isVisible = isElementVisible(node);
+            const isTop = isTopElement(node);
+
+            nodeData.isInteractive = isInteractive;
+            nodeData.isVisible = isVisible;
+            nodeData.isTopElement = isTop;
+
+            // Highlight if element meets all criteria and highlighting is enabled
+            if (isInteractive && isVisible && isTop) {
+                nodeData.highlightIndex = highlightIndex++;
+                if (doHighlightElements) {
+                    highlightElement(node, nodeData.highlightIndex, parentIframe);
+                }
+            }
+        }
+
+        // Only add iframeContext if we're inside an iframe
+        // if (parentIframe) {
+        //     nodeData.iframeContext = `iframe[src="${parentIframe.src || ''}"]`;
+        // }
+
+        // Only add shadowRoot field if it exists
+        if (node.shadowRoot) {
+            nodeData.shadowRoot = true;
+        }
+
+        // Handle shadow DOM
+        if (node.shadowRoot) {
+            const shadowChildren = Array.from(node.shadowRoot.childNodes).map(child =>
+                buildDomTree(child, parentIframe)
+            );
+            nodeData.children.push(...shadowChildren);
+        }
+
+        // Handle iframes
+        if (node.tagName === 'IFRAME') {
+            try {
+                const iframeDoc = node.contentDocument || node.contentWindow.document;
+                if (iframeDoc) {
+                    const iframeChildren = Array.from(iframeDoc.body.childNodes).map(child =>
+                        buildDomTree(child, node)
+                    );
+                    nodeData.children.push(...iframeChildren);
+                }
+            } catch (e) {
+                console.warn('Unable to access iframe:', node);
+            }
+        } else {
+            const children = Array.from(node.childNodes).map(child =>
+                buildDomTree(child, parentIframe)
+            );
+            nodeData.children.push(...children);
+        }
+
+        return nodeData;
+    }
+
+
+    return buildDomTree(document.body);
+}

lumivor/dom/history_tree_processor/service.py

@@ -0,0 +1,112 @@
+import hashlib
+from dataclasses import dataclass
+from typing import Optional
+
+from lumivor.dom.history_tree_processor.view import DOMHistoryElement, HashedDomElement
+from lumivor.dom.views import DOMElementNode
+
+
+class HistoryTreeProcessor:
+    """ "
+    Operations on the DOM elements
+
+    @dev be careful - text nodes can change even if elements stay the same
+    """
+
+    @staticmethod
+    def convert_dom_element_to_history_element(dom_element: DOMElementNode) -> DOMHistoryElement:
+        parent_branch_path = HistoryTreeProcessor._get_parent_branch_path(
+            dom_element)
+        return DOMHistoryElement(
+            dom_element.tag_name,
+            dom_element.xpath,
+            dom_element.highlight_index,
+            parent_branch_path,
+            dom_element.attributes,
+            dom_element.shadow_root,
+        )
+
+    @staticmethod
+    def find_history_element_in_tree(
+            dom_history_element: DOMHistoryElement, tree: DOMElementNode
+    ) -> Optional[DOMElementNode]:
+        hashed_dom_history_element = HistoryTreeProcessor._hash_dom_history_element(
+            dom_history_element
+        )
+
+        def process_node(node: DOMElementNode):
+            if node.highlight_index is not None:
+                hashed_node = HistoryTreeProcessor._hash_dom_element(node)
+                if hashed_node == hashed_dom_history_element:
+                    return node
+            for child in node.children:
+                if isinstance(child, DOMElementNode):
+                    result = process_node(child)
+                    if result is not None:
+                        return result
+            return None
+
+        return process_node(tree)
+
+    @staticmethod
+    def compare_history_element_and_dom_element(
+            dom_history_element: DOMHistoryElement, dom_element: DOMElementNode
+    ) -> bool:
+        hashed_dom_history_element = HistoryTreeProcessor._hash_dom_history_element(
+            dom_history_element
+        )
+        hashed_dom_element = HistoryTreeProcessor._hash_dom_element(
+            dom_element)
+
+        return hashed_dom_history_element == hashed_dom_element
+
+    @staticmethod
+    def _hash_dom_history_element(dom_history_element: DOMHistoryElement) -> HashedDomElement:
+        branch_path_hash = HistoryTreeProcessor._parent_branch_path_hash(
+            dom_history_element.entire_parent_branch_path
+        )
+        attributes_hash = HistoryTreeProcessor._attributes_hash(
+            dom_history_element.attributes)
+
+        return HashedDomElement(branch_path_hash, attributes_hash)
+
+    @staticmethod
+    def _hash_dom_element(dom_element: DOMElementNode) -> HashedDomElement:
+        parent_branch_path = HistoryTreeProcessor._get_parent_branch_path(
+            dom_element)
+        branch_path_hash = HistoryTreeProcessor._parent_branch_path_hash(
+            parent_branch_path)
+        attributes_hash = HistoryTreeProcessor._attributes_hash(
+            dom_element.attributes)
+        # text_hash = DomTreeProcessor._text_hash(dom_element)
+
+        return HashedDomElement(branch_path_hash, attributes_hash)
+
+    @staticmethod
+    def _get_parent_branch_path(dom_element: DOMElementNode) -> list[str]:
+        parents: list[DOMElementNode] = []
+        current_element: DOMElementNode = dom_element
+        while current_element.parent is not None:
+            parents.append(current_element)
+            current_element = current_element.parent
+
+        parents.reverse()
+
+        return [parent.tag_name for parent in parents]
+
+    @staticmethod
+    def _parent_branch_path_hash(parent_branch_path: list[str]) -> str:
+        parent_branch_path_string = '/'.join(parent_branch_path)
+        return hashlib.sha256(parent_branch_path_string.encode()).hexdigest()
+
+    @staticmethod
+    def _attributes_hash(attributes: dict[str, str]) -> str:
+        attributes_string = ''.join(
+            f'{key}={value}' for key, value in attributes.items())
+        return hashlib.sha256(attributes_string.encode()).hexdigest()
+
+    @staticmethod
+    def _text_hash(dom_element: DOMElementNode) -> str:
+        """ """
+        text_string = dom_element.get_all_text_till_next_clickable_element()
+        return hashlib.sha256(text_string.encode()).hexdigest()

lumivor/dom/history_tree_processor/view.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class HashedDomElement:
+	"""
+	Hash of the dom element to be used as a unique identifier
+	"""
+
+	branch_path_hash: str
+	attributes_hash: str
+	# text_hash: str
+
+
+@dataclass
+class DOMHistoryElement:
+	tag_name: str
+	xpath: str
+	highlight_index: Optional[int]
+	entire_parent_branch_path: list[str]
+	attributes: dict[str, str]
+	shadow_root: bool = False
+
+	def to_dict(self) -> dict:
+		return {
+			'tag_name': self.tag_name,
+			'xpath': self.xpath,
+			'highlight_index': self.highlight_index,
+			'entire_parent_branch_path': self.entire_parent_branch_path,
+			'attributes': self.attributes,
+			'shadow_root': self.shadow_root,
+		}

lumivor/dom/service.py

@@ -0,0 +1,100 @@
+import logging
+from importlib import resources
+from typing import Optional
+
+from playwright.async_api import Page
+
+from lumivor.dom.views import (
+    DOMBaseNode,
+    DOMElementNode,
+    DOMState,
+    DOMTextNode,
+    SelectorMap,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DomService:
+    def __init__(self, page: Page):
+        self.page = page
+        self.xpath_cache = {}
+
+    # region - Clickable elements
+    async def get_clickable_elements(self, highlight_elements: bool = True) -> DOMState:
+        element_tree = await self._build_dom_tree(highlight_elements)
+        selector_map = self._create_selector_map(element_tree)
+
+        return DOMState(element_tree=element_tree, selector_map=selector_map)
+
+    async def _build_dom_tree(self, highlight_elements: bool) -> DOMElementNode:
+        js_code = resources.read_text('lumivor.dom', 'buildDomTree.js')
+
+        eval_page = await self.page.evaluate(
+            js_code, [highlight_elements]
+        )  # This is quite big, so be careful
+        html_to_dict = self._parse_node(eval_page)
+
+        if html_to_dict is None or not isinstance(html_to_dict, DOMElementNode):
+            raise ValueError('Failed to parse HTML to dictionary')
+
+        return html_to_dict
+
+    def _create_selector_map(self, element_tree: DOMElementNode) -> SelectorMap:
+        selector_map = {}
+
+        def process_node(node: DOMBaseNode):
+            if isinstance(node, DOMElementNode):
+                if node.highlight_index is not None:
+                    selector_map[node.highlight_index] = node
+
+                for child in node.children:
+                    process_node(child)
+
+        process_node(element_tree)
+        return selector_map
+
+    def _parse_node(
+            self,
+            node_data: dict,
+            parent: Optional[DOMElementNode] = None,
+    ) -> Optional[DOMBaseNode]:
+        if not node_data:
+            return None
+
+        if node_data.get('type') == 'TEXT_NODE':
+            text_node = DOMTextNode(
+                text=node_data['text'],
+                is_visible=node_data['isVisible'],
+                parent=parent,
+            )
+
+            return text_node
+
+        tag_name = node_data['tagName']
+
+        element_node = DOMElementNode(
+            tag_name=tag_name,
+            xpath=node_data['xpath'],
+            attributes=node_data.get('attributes', {}),
+            children=[],  # Initialize empty, will fill later
+            is_visible=node_data.get('isVisible', False),
+            is_interactive=node_data.get('isInteractive', False),
+            is_top_element=node_data.get('isTopElement', False),
+            highlight_index=node_data.get('highlightIndex'),
+            shadow_root=node_data.get('shadowRoot', False),
+            parent=parent,
+        )
+
+        children: list[DOMBaseNode] = []
+        for child in node_data.get('children', []):
+            if child is not None:
+                child_node = self._parse_node(child, parent=element_node)
+                if child_node is not None:
+                    children.append(child_node)
+
+        element_node.children = children
+
+        return element_node
+
+    # endregion