optexity-browser-use 0.9.5__py3-none-any.whl

browser_use/llm/anthropic/serializer.py

@@ -0,0 +1,312 @@
+import json
+from typing import overload
+
+from anthropic.types import (
+	Base64ImageSourceParam,
+	CacheControlEphemeralParam,
+	ImageBlockParam,
+	MessageParam,
+	TextBlockParam,
+	ToolUseBlockParam,
+	URLImageSourceParam,
+)
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SupportedImageMediaType,
+	SystemMessage,
+	UserMessage,
+)
+
+NonSystemMessage = UserMessage | AssistantMessage
+
+
+class AnthropicMessageSerializer:
+	"""Serializer for converting between custom message types and Anthropic message param types."""
+
+	@staticmethod
+	def _is_base64_image(url: str) -> bool:
+		"""Check if the URL is a base64 encoded image."""
+		return url.startswith('data:image/')
+
+	@staticmethod
+	def _parse_base64_url(url: str) -> tuple[SupportedImageMediaType, str]:
+		"""Parse a base64 data URL to extract media type and data."""
+		# Format: data:image/jpeg;base64,<data>
+		if not url.startswith('data:'):
+			raise ValueError(f'Invalid base64 URL: {url}')
+
+		header, data = url.split(',', 1)
+		media_type = header.split(';')[0].replace('data:', '')
+
+		# Ensure it's a supported media type
+		supported_types = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']
+		if media_type not in supported_types:
+			# Default to jpeg if not recognized
+			media_type = 'image/jpeg'
+
+		return media_type, data  # type: ignore
+
+	@staticmethod
+	def _serialize_cache_control(use_cache: bool) -> CacheControlEphemeralParam | None:
+		"""Serialize cache control."""
+		if use_cache:
+			return CacheControlEphemeralParam(type='ephemeral')
+		return None
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam, use_cache: bool) -> TextBlockParam:
+		"""Convert a text content part to Anthropic's TextBlockParam."""
+		return TextBlockParam(
+			text=part.text, type='text', cache_control=AnthropicMessageSerializer._serialize_cache_control(use_cache)
+		)
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> ImageBlockParam:
+		"""Convert an image content part to Anthropic's ImageBlockParam."""
+		url = part.image_url.url
+
+		if AnthropicMessageSerializer._is_base64_image(url):
+			# Handle base64 encoded images
+			media_type, data = AnthropicMessageSerializer._parse_base64_url(url)
+			return ImageBlockParam(
+				source=Base64ImageSourceParam(
+					data=data,
+					media_type=media_type,
+					type='base64',
+				),
+				type='image',
+			)
+		else:
+			# Handle URL images
+			return ImageBlockParam(source=URLImageSourceParam(url=url, type='url'), type='image')
+
+	@staticmethod
+	def _serialize_content_to_str(
+		content: str | list[ContentPartTextParam], use_cache: bool = False
+	) -> list[TextBlockParam] | str:
+		"""Serialize content to a string."""
+		cache_control = AnthropicMessageSerializer._serialize_cache_control(use_cache)
+
+		if isinstance(content, str):
+			if cache_control:
+				return [TextBlockParam(text=content, type='text', cache_control=cache_control)]
+			else:
+				return content
+
+		serialized_blocks: list[TextBlockParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_text(part, use_cache))
+
+		return serialized_blocks
+
+	@staticmethod
+	def _serialize_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+		use_cache: bool = False,
+	) -> str | list[TextBlockParam | ImageBlockParam]:
+		"""Serialize content to Anthropic format."""
+		if isinstance(content, str):
+			if use_cache:
+				return [TextBlockParam(text=content, type='text', cache_control=CacheControlEphemeralParam(type='ephemeral'))]
+			else:
+				return content
+
+		serialized_blocks: list[TextBlockParam | ImageBlockParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_text(part, use_cache))
+			elif part.type == 'image_url':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_image(part))
+
+		return serialized_blocks
+
+	@staticmethod
+	def _serialize_tool_calls_to_content(tool_calls, use_cache: bool = False) -> list[ToolUseBlockParam]:
+		"""Convert tool calls to Anthropic's ToolUseBlockParam format."""
+		blocks: list[ToolUseBlockParam] = []
+		for tool_call in tool_calls:
+			# Parse the arguments JSON string to object
+
+			try:
+				input_obj = json.loads(tool_call.function.arguments)
+			except json.JSONDecodeError:
+				# If arguments aren't valid JSON, use as string
+				input_obj = {'arguments': tool_call.function.arguments}
+
+			blocks.append(
+				ToolUseBlockParam(
+					id=tool_call.id,
+					input=input_obj,
+					name=tool_call.function.name,
+					type='tool_use',
+					cache_control=AnthropicMessageSerializer._serialize_cache_control(use_cache),
+				)
+			)
+		return blocks
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> MessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> SystemMessage: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> MessageParam: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> MessageParam | SystemMessage:
+		"""Serialize a custom message to an Anthropic MessageParam.
+
+		Note: Anthropic doesn't have a 'system' role. System messages should be
+		handled separately as the system parameter in the API call, not as a message.
+		If a SystemMessage is passed here, it will be converted to a user message.
+		"""
+		if isinstance(message, UserMessage):
+			content = AnthropicMessageSerializer._serialize_content(message.content, use_cache=message.cache)
+			return MessageParam(role='user', content=content)
+
+		elif isinstance(message, SystemMessage):
+			# Anthropic doesn't have system messages in the messages array
+			# System prompts are passed separately. Convert to user message.
+			return message
+
+		elif isinstance(message, AssistantMessage):
+			# Handle content and tool calls
+			blocks: list[TextBlockParam | ToolUseBlockParam] = []
+
+			# Add content blocks if present
+			if message.content is not None:
+				if isinstance(message.content, str):
+					blocks.append(
+						TextBlockParam(
+							text=message.content,
+							type='text',
+							cache_control=AnthropicMessageSerializer._serialize_cache_control(message.cache),
+						)
+					)
+				else:
+					# Process content parts (text and refusal)
+					for part in message.content:
+						if part.type == 'text':
+							blocks.append(AnthropicMessageSerializer._serialize_content_part_text(part, use_cache=message.cache))
+						# # Note: Anthropic doesn't have a specific refusal block type,
+						# # so we convert refusals to text blocks
+						# elif part.type == 'refusal':
+						# 	blocks.append(TextBlockParam(text=f'[Refusal] {part.refusal}', type='text'))
+
+			# Add tool use blocks if present
+			if message.tool_calls:
+				tool_blocks = AnthropicMessageSerializer._serialize_tool_calls_to_content(
+					message.tool_calls, use_cache=message.cache
+				)
+				blocks.extend(tool_blocks)
+
+			# If no content or tool calls, add empty text block
+			# (Anthropic requires at least one content block)
+			if not blocks:
+				blocks.append(
+					TextBlockParam(
+						text='', type='text', cache_control=AnthropicMessageSerializer._serialize_cache_control(message.cache)
+					)
+				)
+
+			# If caching is enabled or we have multiple blocks, return blocks as-is
+			# Otherwise, simplify single text blocks to plain string
+			if message.cache or len(blocks) > 1:
+				content = blocks
+			else:
+				# Only simplify when no caching and single block
+				single_block = blocks[0]
+				if single_block['type'] == 'text' and not single_block.get('cache_control'):
+					content = single_block['text']
+				else:
+					content = blocks
+
+			return MessageParam(
+				role='assistant',
+				content=content,
+			)
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def _clean_cache_messages(messages: list[NonSystemMessage]) -> list[NonSystemMessage]:
+		"""Clean cache settings so only the last cache=True message remains cached.
+
+		Because of how Claude caching works, only the last cache message matters.
+		This method automatically removes cache=True from all messages except the last one.
+
+		Args:
+			messages: List of non-system messages to clean
+
+		Returns:
+			List of messages with cleaned cache settings
+		"""
+		if not messages:
+			return messages
+
+		# Create a copy to avoid modifying the original
+		cleaned_messages = [msg.model_copy(deep=True) for msg in messages]
+
+		# Find the last message with cache=True
+		last_cache_index = -1
+		for i in range(len(cleaned_messages) - 1, -1, -1):
+			if cleaned_messages[i].cache:
+				last_cache_index = i
+				break
+
+		# If we found a cached message, disable cache for all others
+		if last_cache_index != -1:
+			for i, msg in enumerate(cleaned_messages):
+				if i != last_cache_index and msg.cache:
+					# Set cache to False for all messages except the last cached one
+					msg.cache = False
+
+		return cleaned_messages
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> tuple[list[MessageParam], list[TextBlockParam] | str | None]:
+		"""Serialize a list of messages, extracting any system message.
+
+		Returns:
+		    A tuple of (messages, system_message) where system_message is extracted
+		    from any SystemMessage in the list.
+		"""
+		messages = [m.model_copy(deep=True) for m in messages]
+
+		# Separate system messages from normal messages
+		normal_messages: list[NonSystemMessage] = []
+		system_message: SystemMessage | None = None
+
+		for message in messages:
+			if isinstance(message, SystemMessage):
+				system_message = message
+			else:
+				normal_messages.append(message)
+
+		# Clean cache messages so only the last cache=True message remains cached
+		normal_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages)
+
+		# Serialize normal messages
+		serialized_messages: list[MessageParam] = []
+		for message in normal_messages:
+			serialized_messages.append(AnthropicMessageSerializer.serialize(message))
+
+		# Serialize system message
+		serialized_system_message: list[TextBlockParam] | str | None = None
+		if system_message:
+			serialized_system_message = AnthropicMessageSerializer._serialize_content_to_str(
+				system_message.content, use_cache=system_message.cache
+			)
+
+		return serialized_messages, serialized_system_message

browser_use/llm/aws/__init__.py

@@ -0,0 +1,36 @@
+from typing import TYPE_CHECKING
+
+# Type stubs for lazy imports
+if TYPE_CHECKING:
+	from browser_use.llm.aws.chat_anthropic import ChatAnthropicBedrock
+	from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+
+# Lazy imports mapping for AWS chat models
+_LAZY_IMPORTS = {
+	'ChatAnthropicBedrock': ('browser_use.llm.aws.chat_anthropic', 'ChatAnthropicBedrock'),
+	'ChatAWSBedrock': ('browser_use.llm.aws.chat_bedrock', 'ChatAWSBedrock'),
+}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism for AWS chat models."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			module = import_module(module_path)
+			attr = getattr(module, attr_name)
+			# Cache the imported attribute in the module's globals
+			globals()[name] = attr
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'ChatAWSBedrock',
+	'ChatAnthropicBedrock',
+]

browser_use/llm/aws/chat_anthropic.py

@@ -0,0 +1,242 @@
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypeVar, overload
+
+from anthropic import (
+	APIConnectionError,
+	APIStatusError,
+	AsyncAnthropicBedrock,
+	RateLimitError,
+	omit,
+)
+from anthropic.types import CacheControlEphemeralParam, Message, ToolParam
+from anthropic.types.text_block import TextBlock
+from anthropic.types.tool_choice_tool_param import ToolChoiceToolParam
+from pydantic import BaseModel
+
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer
+from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+if TYPE_CHECKING:
+	from boto3.session import Session  # pyright: ignore
+
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAnthropicBedrock(ChatAWSBedrock):
+	"""
+	AWS Bedrock Anthropic Claude chat model.
+
+	This is a convenience class that provides Claude-specific defaults
+	for the AWS Bedrock service. It inherits all functionality from
+	ChatAWSBedrock but sets Anthropic Claude as the default model.
+	"""
+
+	# Anthropic Claude specific defaults
+	model: str = 'anthropic.claude-3-5-sonnet-20240620-v1:0'
+	max_tokens: int = 8192
+	temperature: float | None = None
+	top_p: float | None = None
+	top_k: int | None = None
+	stop_sequences: list[str] | None = None
+	seed: int | None = None
+
+	# AWS credentials and configuration
+	aws_access_key: str | None = None
+	aws_secret_key: str | None = None
+	aws_session_token: str | None = None
+	aws_region: str | None = None
+	session: 'Session | None' = None
+
+	# Client initialization parameters
+	max_retries: int = 10
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'anthropic_bedrock'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for Bedrock."""
+		client_params: dict[str, Any] = {}
+
+		if self.session:
+			credentials = self.session.get_credentials()
+			client_params.update(
+				{
+					'aws_access_key': credentials.access_key,
+					'aws_secret_key': credentials.secret_key,
+					'aws_session_token': credentials.token,
+					'aws_region': self.session.region_name,
+				}
+			)
+		else:
+			# Use individual credentials
+			if self.aws_access_key:
+				client_params['aws_access_key'] = self.aws_access_key
+			if self.aws_secret_key:
+				client_params['aws_secret_key'] = self.aws_secret_key
+			if self.aws_region:
+				client_params['aws_region'] = self.aws_region
+			if self.aws_session_token:
+				client_params['aws_session_token'] = self.aws_session_token
+
+		# Add optional parameters
+		if self.max_retries:
+			client_params['max_retries'] = self.max_retries
+		if self.default_headers:
+			client_params['default_headers'] = self.default_headers
+		if self.default_query:
+			client_params['default_query'] = self.default_query
+
+		return client_params
+
+	def _get_client_params_for_invoke(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for invoke."""
+		client_params = {}
+
+		if self.temperature is not None:
+			client_params['temperature'] = self.temperature
+		if self.max_tokens is not None:
+			client_params['max_tokens'] = self.max_tokens
+		if self.top_p is not None:
+			client_params['top_p'] = self.top_p
+		if self.top_k is not None:
+			client_params['top_k'] = self.top_k
+		if self.seed is not None:
+			client_params['seed'] = self.seed
+		if self.stop_sequences is not None:
+			client_params['stop_sequences'] = self.stop_sequences
+
+		return client_params
+
+	def get_client(self) -> AsyncAnthropicBedrock:
+		"""
+		Returns an AsyncAnthropicBedrock client.
+
+		Returns:
+			AsyncAnthropicBedrock: An instance of the AsyncAnthropicBedrock client.
+		"""
+		client_params = self._get_client_params()
+		return AsyncAnthropicBedrock(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: Message) -> ChatInvokeUsage | None:
+		"""Extract usage information from the response."""
+		usage = ChatInvokeUsage(
+			prompt_tokens=response.usage.input_tokens
+			+ (
+				response.usage.cache_read_input_tokens or 0
+			),  # Total tokens in Anthropic are a bit fucked, you have to add cached tokens to the prompt tokens
+			completion_tokens=response.usage.output_tokens,
+			total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+			prompt_cached_tokens=response.usage.cache_read_input_tokens,
+			prompt_cache_creation_tokens=response.usage.cache_creation_input_tokens,
+			prompt_image_tokens=None,
+		)
+		return usage
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		anthropic_messages, system_prompt = AnthropicMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				# Normal completion without structured output
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					system=system_prompt or omit,
+					**self._get_client_params_for_invoke(),
+				)
+
+				usage = self._get_usage(response)
+
+				# Extract text from the first content block
+				first_content = response.content[0]
+				if isinstance(first_content, TextBlock):
+					response_text = first_content.text
+				else:
+					# If it's not a text block, convert to string
+					response_text = str(first_content)
+
+				return ChatInvokeCompletion(
+					completion=response_text,
+					usage=usage,
+				)
+
+			else:
+				# Use tool calling for structured output
+				# Create a tool that represents the output format
+				tool_name = output_format.__name__
+				schema = output_format.model_json_schema()
+
+				# Remove title from schema if present (Anthropic doesn't like it in parameters)
+				if 'title' in schema:
+					del schema['title']
+
+				tool = ToolParam(
+					name=tool_name,
+					description=f'Extract information in the format of {tool_name}',
+					input_schema=schema,
+					cache_control=CacheControlEphemeralParam(type='ephemeral'),
+				)
+
+				# Force the model to use this tool
+				tool_choice = ToolChoiceToolParam(type='tool', name=tool_name)
+
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					tools=[tool],
+					system=system_prompt or omit,
+					tool_choice=tool_choice,
+					**self._get_client_params_for_invoke(),
+				)
+
+				usage = self._get_usage(response)
+
+				# Extract the tool use block
+				for content_block in response.content:
+					if hasattr(content_block, 'type') and content_block.type == 'tool_use':
+						# Parse the tool input as the structured output
+						try:
+							return ChatInvokeCompletion(completion=output_format.model_validate(content_block.input), usage=usage)
+						except Exception as e:
+							# If validation fails, try to parse it as JSON first
+							if isinstance(content_block.input, str):
+								data = json.loads(content_block.input)
+								return ChatInvokeCompletion(
+									completion=output_format.model_validate(data),
+									usage=usage,
+								)
+							raise e
+
+				# If no tool use block found, raise an error
+				raise ValueError('Expected tool use in response but none found')
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=e.message, model=self.name) from e
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e