optexity-browser-use 0.9.5__py3-none-any.whl

browser_use/dom/serializer/eval_serializer.py

@@ -0,0 +1,478 @@
+# @file purpose: Concise evaluation serializer for DOM trees - optimized for LLM query writing
+
+
+from browser_use.dom.utils import cap_text_length
+from browser_use.dom.views import (
+	EnhancedDOMTreeNode,
+	NodeType,
+	SimplifiedNode,
+)
+
+# Critical attributes for query writing and form interaction
+# NOTE: Removed 'id' and 'class' to force more robust structural selectors
+EVAL_KEY_ATTRIBUTES = [
+	'id',  # Removed - can have special chars, forces structural selectors
+	'class',  # Removed - can have special chars like +, forces structural selectors
+	'name',
+	'type',
+	'placeholder',
+	'aria-label',
+	'role',
+	'value',
+	# 'href',
+	'data-testid',
+	'alt',  # for images
+	'title',  # useful for tooltips/link context
+	# State attributes (critical for form interaction)
+	'checked',
+	'selected',
+	'disabled',
+	'required',
+	'readonly',
+	# ARIA states
+	'aria-expanded',
+	'aria-pressed',
+	'aria-checked',
+	'aria-selected',
+	'aria-invalid',
+	# Validation attributes (help agents avoid brute force)
+	'pattern',
+	'min',
+	'max',
+	'minlength',
+	'maxlength',
+	'step',
+	'aria-valuemin',
+	'aria-valuemax',
+	'aria-valuenow',
+]
+
+# Semantic elements that should always be shown
+SEMANTIC_ELEMENTS = {
+	'html',  # Always show document root
+	'body',  # Always show body
+	'h1',
+	'h2',
+	'h3',
+	'h4',
+	'h5',
+	'h6',
+	'a',
+	'button',
+	'input',
+	'textarea',
+	'select',
+	'form',
+	'label',
+	'nav',
+	'header',
+	'footer',
+	'main',
+	'article',
+	'section',
+	'table',
+	'thead',
+	'tbody',
+	'tr',
+	'th',
+	'td',
+	'ul',
+	'ol',
+	'li',
+	'img',
+	'iframe',
+	'video',
+	'audio',
+}
+
+# Container elements that can be collapsed if they only wrap one child
+COLLAPSIBLE_CONTAINERS = {'div', 'span', 'section', 'article'}
+
+# SVG child elements to skip (decorative only, no interaction value)
+SVG_ELEMENTS = {
+	'path',
+	'rect',
+	'g',
+	'circle',
+	'ellipse',
+	'line',
+	'polyline',
+	'polygon',
+	'use',
+	'defs',
+	'clipPath',
+	'mask',
+	'pattern',
+	'image',
+	'text',
+	'tspan',
+}
+
+
+class DOMEvalSerializer:
+	"""Ultra-concise DOM serializer for quick LLM query writing."""
+
+	@staticmethod
+	def serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:
+		"""
+		Serialize complete DOM tree structure for LLM understanding.
+
+		Strategy:
+		- Show ALL elements to preserve DOM structure
+		- Non-interactive elements show just tag name
+		- Interactive elements show full attributes + [index]
+		- Self-closing tags only (no closing tags)
+		"""
+		if not node:
+			return ''
+
+		# Skip excluded nodes but process children
+		if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:
+			return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+		# Skip nodes marked as should_display=False
+		if not node.should_display:
+			return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+		formatted_text = []
+		depth_str = depth * '\t'
+
+		if node.original_node.node_type == NodeType.ELEMENT_NODE:
+			tag = node.original_node.tag_name.lower()
+			is_visible = node.original_node.snapshot_node and node.original_node.is_visible
+
+			# Container elements that should be shown even if invisible (might have visible children)
+			container_tags = {'html', 'body', 'div', 'main', 'section', 'article', 'aside', 'header', 'footer', 'nav'}
+
+			# Skip invisible elements UNLESS they're containers or iframes (which might have visible children)
+			if not is_visible and tag not in container_tags and tag not in ['iframe', 'frame']:
+				return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+			# Special handling for iframes - show them with their content
+			if tag in ['iframe', 'frame']:
+				return DOMEvalSerializer._serialize_iframe(node, include_attributes, depth)
+
+			# Skip SVG elements entirely - they're just decorative graphics with no interaction value
+			# Show the <svg> tag itself to indicate graphics, but don't recurse into children
+			if tag == 'svg':
+				line = f'{depth_str}'
+				# Add [i_X] for interactive SVG elements only
+				if node.is_interactive:
+					line += f'[i_{node.original_node.backend_node_id}] '
+				line += '<svg'
+				attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+				if attributes_str:
+					line += f' {attributes_str}'
+				line += ' /> <!-- SVG content collapsed -->'
+				return line
+
+			# Skip SVG child elements entirely (path, rect, g, circle, etc.)
+			if tag in SVG_ELEMENTS:
+				return ''
+
+			# Build compact attributes string
+			attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+
+			# Decide if this element should be shown
+			is_semantic = tag in SEMANTIC_ELEMENTS
+			has_useful_attrs = bool(attributes_str)
+			has_text_content = DOMEvalSerializer._has_direct_text(node)
+			has_children = len(node.children) > 0
+
+			# Build compact element representation
+			line = f'{depth_str}'
+			# Add backend node ID notation - [i_X] for interactive elements only
+			if node.is_interactive:
+				line += f'[i_{node.original_node.backend_node_id}] '
+			# Non-interactive elements don't get an index notation
+			line += f'<{tag}'
+
+			if attributes_str:
+				line += f' {attributes_str}'
+
+			# Add scroll info if element is scrollable
+			if node.original_node.should_show_scroll_info:
+				scroll_text = node.original_node.get_scroll_info_text()
+				if scroll_text:
+					line += f' scroll="{scroll_text}"'
+
+			# Add inline text if present (keep it on same line for compactness)
+			inline_text = DOMEvalSerializer._get_inline_text(node)
+
+			# For containers (html, body, div, etc.), always show children even if there's inline text
+			# For other elements, inline text replaces children (more compact)
+			is_container = tag in container_tags
+
+			if inline_text and not is_container:
+				line += f'>{inline_text}'
+			else:
+				line += ' />'
+
+			formatted_text.append(line)
+
+			# Process children (always for containers, only if no inline_text for others)
+			if has_children and (is_container or not inline_text):
+				children_text = DOMEvalSerializer._serialize_children(node, include_attributes, depth + 1)
+				if children_text:
+					formatted_text.append(children_text)
+
+		elif node.original_node.node_type == NodeType.TEXT_NODE:
+			# Text nodes are handled inline with their parent
+			pass
+
+		elif node.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+			# Shadow DOM - just show children directly with minimal marker
+			if node.children:
+				formatted_text.append(f'{depth_str}#shadow')
+				children_text = DOMEvalSerializer._serialize_children(node, include_attributes, depth + 1)
+				if children_text:
+					formatted_text.append(children_text)
+
+		return '\n'.join(formatted_text)
+
+	@staticmethod
+	def _serialize_children(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+		"""Helper to serialize all children of a node."""
+		children_output = []
+
+		# Check if parent is a list container (ul, ol)
+		is_list_container = node.original_node.node_type == NodeType.ELEMENT_NODE and node.original_node.tag_name.lower() in [
+			'ul',
+			'ol',
+		]
+
+		# Track list items and consecutive links
+		li_count = 0
+		max_list_items = 50
+		consecutive_link_count = 0
+		max_consecutive_links = 50
+		total_links_skipped = 0
+
+		for child in node.children:
+			# Get tag name for this child
+			current_tag = None
+			if child.original_node.node_type == NodeType.ELEMENT_NODE:
+				current_tag = child.original_node.tag_name.lower()
+
+			# If we're in a list container and this child is an li element
+			if is_list_container and current_tag == 'li':
+				li_count += 1
+				# Skip li elements after the 5th one
+				if li_count > max_list_items:
+					continue
+
+			# Track consecutive anchor tags (links)
+			if current_tag == 'a':
+				consecutive_link_count += 1
+				# Skip links after the 5th consecutive one
+				if consecutive_link_count > max_consecutive_links:
+					total_links_skipped += 1
+					continue
+			else:
+				# Reset counter when we hit a non-link element
+				# But first add truncation message if we skipped links
+				if total_links_skipped > 0:
+					depth_str = depth * '\t'
+					children_output.append(f'{depth_str}... ({total_links_skipped} more links in this list)')
+					total_links_skipped = 0
+				consecutive_link_count = 0
+
+			child_text = DOMEvalSerializer.serialize_tree(child, include_attributes, depth)
+			if child_text:
+				children_output.append(child_text)
+
+		# Add truncation message if we skipped items at the end
+		if is_list_container and li_count > max_list_items:
+			depth_str = depth * '\t'
+			children_output.append(
+				f'{depth_str}... ({li_count - max_list_items} more items in this list (truncated) use evaluate to get more.'
+			)
+
+		# Add truncation message for links if we skipped any at the end
+		if total_links_skipped > 0:
+			depth_str = depth * '\t'
+			children_output.append(
+				f'{depth_str}... ({total_links_skipped} more links in this list) (truncated) use evaluate to get more.'
+			)
+
+		return '\n'.join(children_output)
+
+	@staticmethod
+	def _build_compact_attributes(node: EnhancedDOMTreeNode) -> str:
+		"""Build ultra-compact attributes string with only key attributes."""
+		attrs = []
+
+		# Prioritize attributes that help with query writing
+		if node.attributes:
+			for attr in EVAL_KEY_ATTRIBUTES:
+				if attr in node.attributes:
+					value = str(node.attributes[attr]).strip()
+					if not value:
+						continue
+
+					# Special handling for different attributes
+					if attr == 'class':
+						# For class, limit to first 2 classes to save space
+						classes = value.split()[:3]
+						value = ' '.join(classes)
+					elif attr == 'href':
+						# For href, cap at 20 chars to save space
+						value = cap_text_length(value, 80)
+					else:
+						# Cap at 25 chars for other attributes
+						value = cap_text_length(value, 80)
+
+					attrs.append(f'{attr}="{value}"')
+
+		# Note: We intentionally don't add role from ax_node here because:
+		# 1. If role is explicitly set in HTML, it's already captured above via EVAL_KEY_ATTRIBUTES
+		# 2. Inferred roles from AX tree (like link, listitem, LineBreak) are redundant with the tag name
+		# 3. This reduces noise - <a href="..." role="link"> is redundant, we already know <a> is a link
+
+		return ' '.join(attrs)
+
+	@staticmethod
+	def _has_direct_text(node: SimplifiedNode) -> bool:
+		"""Check if node has direct text children (not nested in other elements)."""
+		for child in node.children:
+			if child.original_node.node_type == NodeType.TEXT_NODE:
+				text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+				if len(text) > 1:
+					return True
+		return False
+
+	@staticmethod
+	def _get_inline_text(node: SimplifiedNode) -> str:
+		"""Get text content to display inline (max 40 chars)."""
+		text_parts = []
+		for child in node.children:
+			if child.original_node.node_type == NodeType.TEXT_NODE:
+				text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+				if text and len(text) > 1:
+					text_parts.append(text)
+
+		if not text_parts:
+			return ''
+
+		combined = ' '.join(text_parts)
+		return cap_text_length(combined, 80)
+
+	@staticmethod
+	def _serialize_iframe(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+		"""Handle iframe serialization with content document."""
+		formatted_text = []
+		depth_str = depth * '\t'
+		tag = node.original_node.tag_name.lower()
+
+		# Build minimal iframe marker with key attributes
+		attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+		line = f'{depth_str}<{tag}'
+		if attributes_str:
+			line += f' {attributes_str}'
+
+		# Add scroll info for iframe content
+		if node.original_node.should_show_scroll_info:
+			scroll_text = node.original_node.get_scroll_info_text()
+			if scroll_text:
+				line += f' scroll="{scroll_text}"'
+
+		line += ' />'
+		formatted_text.append(line)
+
+		# If iframe has content document, serialize its content
+		if node.original_node.content_document:
+			# Add marker for iframe content
+			formatted_text.append(f'{depth_str}\t#iframe-content')
+
+			# Process content document children
+			for child_node in node.original_node.content_document.children_nodes or []:
+				# Process html documents
+				if child_node.tag_name.lower() == 'html':
+					# Find and serialize body content only (skip head)
+					for html_child in child_node.children:
+						if html_child.tag_name.lower() == 'body':
+							for body_child in html_child.children:
+								# Recursively process body children (iframe content)
+								DOMEvalSerializer._serialize_document_node(
+									body_child, formatted_text, include_attributes, depth + 2, is_iframe_content=True
+								)
+							break  # Stop after processing body
+				else:
+					# Not an html element - serialize directly
+					DOMEvalSerializer._serialize_document_node(
+						child_node, formatted_text, include_attributes, depth + 1, is_iframe_content=True
+					)
+
+		return '\n'.join(formatted_text)
+
+	@staticmethod
+	def _serialize_document_node(
+		dom_node: EnhancedDOMTreeNode,
+		output: list[str],
+		include_attributes: list[str],
+		depth: int,
+		is_iframe_content: bool = True,
+	) -> None:
+		"""Helper to serialize a document node without SimplifiedNode wrapper.
+
+		Args:
+			is_iframe_content: If True, be more permissive with visibility checks since
+				iframe content might not have snapshot data from parent page.
+		"""
+		depth_str = depth * '\t'
+
+		if dom_node.node_type == NodeType.ELEMENT_NODE:
+			tag = dom_node.tag_name.lower()
+
+			# For iframe content, be permissive - show all semantic elements even without snapshot data
+			# For regular content, skip invisible elements
+			if is_iframe_content:
+				# Only skip if we have snapshot data AND it's explicitly invisible
+				# If no snapshot data, assume visible (cross-origin iframe content)
+				is_visible = (not dom_node.snapshot_node) or dom_node.is_visible
+			else:
+				# Regular strict visibility check
+				is_visible = dom_node.snapshot_node and dom_node.is_visible
+
+			if not is_visible:
+				return
+
+			# Check if semantic or has useful attributes
+			is_semantic = tag in SEMANTIC_ELEMENTS
+			attributes_str = DOMEvalSerializer._build_compact_attributes(dom_node)
+
+			if not is_semantic and not attributes_str:
+				# Skip but process children
+				for child in dom_node.children:
+					DOMEvalSerializer._serialize_document_node(
+						child, output, include_attributes, depth, is_iframe_content=is_iframe_content
+					)
+				return
+
+			# Build element line
+			line = f'{depth_str}<{tag}'
+			if attributes_str:
+				line += f' {attributes_str}'
+
+			# Get direct text content
+			text_parts = []
+			for child in dom_node.children:
+				if child.node_type == NodeType.TEXT_NODE and child.node_value:
+					text = child.node_value.strip()
+					if text and len(text) > 1:
+						text_parts.append(text)
+
+			if text_parts:
+				combined = ' '.join(text_parts)
+				line += f'>{cap_text_length(combined, 100)}'
+			else:
+				line += ' />'
+
+			output.append(line)
+
+			# Process non-text children
+			for child in dom_node.children:
+				if child.node_type != NodeType.TEXT_NODE:
+					DOMEvalSerializer._serialize_document_node(
+						child, output, include_attributes, depth + 1, is_iframe_content=is_iframe_content
+					)

browser_use/dom/serializer/html_serializer.py

@@ -0,0 +1,212 @@
+# @file purpose: Serializes enhanced DOM trees to HTML format including shadow roots
+
+from browser_use.dom.views import EnhancedDOMTreeNode, NodeType
+
+
+class HTMLSerializer:
+	"""Serializes enhanced DOM trees back to HTML format.
+
+	This serializer reconstructs HTML from the enhanced DOM tree, including:
+	- Shadow DOM content (both open and closed)
+	- Iframe content documents
+	- All attributes and text nodes
+	- Proper HTML structure
+
+	Unlike getOuterHTML which only captures light DOM, this captures the full
+	enhanced tree including shadow roots that are crucial for modern SPAs.
+	"""
+
+	def __init__(self, extract_links: bool = False):
+		"""Initialize the HTML serializer.
+
+		Args:
+			extract_links: If True, preserves all links. If False, removes href attributes.
+		"""
+		self.extract_links = extract_links
+
+	def serialize(self, node: EnhancedDOMTreeNode, depth: int = 0) -> str:
+		"""Serialize an enhanced DOM tree node to HTML.
+
+		Args:
+			node: The enhanced DOM tree node to serialize
+			depth: Current depth for indentation (internal use)
+
+		Returns:
+			HTML string representation of the node and its descendants
+		"""
+		if node.node_type == NodeType.DOCUMENT_NODE:
+			# Process document root - serialize all children
+			parts = []
+			for child in node.children_and_shadow_roots:
+				child_html = self.serialize(child, depth)
+				if child_html:
+					parts.append(child_html)
+			return ''.join(parts)
+
+		elif node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+			# Shadow DOM root - wrap in template with shadowrootmode attribute
+			parts = []
+
+			# Add shadow root opening
+			shadow_type = node.shadow_root_type or 'open'
+			parts.append(f'<template shadowroot="{shadow_type.lower()}">')
+
+			# Serialize shadow children
+			for child in node.children:
+				child_html = self.serialize(child, depth + 1)
+				if child_html:
+					parts.append(child_html)
+
+			# Close shadow root
+			parts.append('</template>')
+
+			return ''.join(parts)
+
+		elif node.node_type == NodeType.ELEMENT_NODE:
+			parts = []
+			tag_name = node.tag_name.lower()
+
+			# Skip non-content elements
+			if tag_name in {'style', 'script', 'head', 'meta', 'link', 'title'}:
+				return ''
+
+			# Skip code tags with display:none - these often contain JSON state for SPAs
+			if tag_name == 'code' and node.attributes:
+				style = node.attributes.get('style', '')
+				# Check if element is hidden (display:none) - likely JSON data
+				if 'display:none' in style.replace(' ', '') or 'display: none' in style:
+					return ''
+				# Also check for bpr-guid IDs (LinkedIn's JSON data pattern)
+				element_id = node.attributes.get('id', '')
+				if 'bpr-guid' in element_id or 'data' in element_id or 'state' in element_id:
+					return ''
+
+			# Skip base64 inline images - these are usually placeholders or tracking pixels
+			if tag_name == 'img' and node.attributes:
+				src = node.attributes.get('src', '')
+				if src.startswith('data:image/'):
+					return ''
+
+			# Opening tag
+			parts.append(f'<{tag_name}')
+
+			# Add attributes
+			if node.attributes:
+				attrs = self._serialize_attributes(node.attributes)
+				if attrs:
+					parts.append(' ' + attrs)
+
+			# Handle void elements (self-closing)
+			void_elements = {
+				'area',
+				'base',
+				'br',
+				'col',
+				'embed',
+				'hr',
+				'img',
+				'input',
+				'link',
+				'meta',
+				'param',
+				'source',
+				'track',
+				'wbr',
+			}
+			if tag_name in void_elements:
+				parts.append(' />')
+				return ''.join(parts)
+
+			parts.append('>')
+
+			# Handle iframe content document
+			if tag_name in {'iframe', 'frame'} and node.content_document:
+				# Serialize iframe content
+				for child in node.content_document.children_nodes or []:
+					child_html = self.serialize(child, depth + 1)
+					if child_html:
+						parts.append(child_html)
+			else:
+				# Serialize shadow roots FIRST (for declarative shadow DOM)
+				if node.shadow_roots:
+					for shadow_root in node.shadow_roots:
+						child_html = self.serialize(shadow_root, depth + 1)
+						if child_html:
+							parts.append(child_html)
+
+				# Then serialize light DOM children (for slot projection)
+				for child in node.children:
+					child_html = self.serialize(child, depth + 1)
+					if child_html:
+						parts.append(child_html)
+
+			# Closing tag
+			parts.append(f'</{tag_name}>')
+
+			return ''.join(parts)
+
+		elif node.node_type == NodeType.TEXT_NODE:
+			# Return text content with basic HTML escaping
+			if node.node_value:
+				return self._escape_html(node.node_value)
+			return ''
+
+		elif node.node_type == NodeType.COMMENT_NODE:
+			# Skip comments to reduce noise
+			return ''
+
+		else:
+			# Unknown node type - skip
+			return ''
+
+	def _serialize_attributes(self, attributes: dict[str, str]) -> str:
+		"""Serialize element attributes to HTML attribute string.
+
+		Args:
+			attributes: Dictionary of attribute names to values
+
+		Returns:
+			HTML attribute string (e.g., 'class="foo" id="bar"')
+		"""
+		parts = []
+		for key, value in attributes.items():
+			# Skip href if not extracting links
+			if not self.extract_links and key == 'href':
+				continue
+
+			# Skip data-* attributes as they often contain JSON payloads
+			# These are used by modern SPAs (React, Vue, Angular) for state management
+			if key.startswith('data-'):
+				continue
+
+			# Handle boolean attributes
+			if value == '' or value is None:
+				parts.append(key)
+			else:
+				# Escape attribute value
+				escaped_value = self._escape_attribute(value)
+				parts.append(f'{key}="{escaped_value}"')
+
+		return ' '.join(parts)
+
+	def _escape_html(self, text: str) -> str:
+		"""Escape HTML special characters in text content.
+
+		Args:
+			text: Raw text content
+
+		Returns:
+			HTML-escaped text
+		"""
+		return text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
+
+	def _escape_attribute(self, value: str) -> str:
+		"""Escape HTML special characters in attribute values.
+
+		Args:
+			value: Raw attribute value
+
+		Returns:
+			HTML-escaped attribute value
+		"""
+		return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#x27;')