aip-agents-binary 0.5.20__py3-none-manylinux_2_31_x86_64.whl

aip_agents/tools/browser_use/streaming.pyi

@@ -0,0 +1,81 @@
+from aip_agents.tools.browser_use.types import StreamingResponse, ToolCallInfo
+from browser_use import Agent
+from typing import Any, Literal
+
+__all__ = ['PROCESSING_MESSAGE', 'TASK_COMPLETED_MESSAGE', 'create_error_response', 'create_step_response', 'generate_step_content', 'generate_thinking_message', 'yield_iframe_activity', 'yield_status_message', 'yield_thinking_marker']
+
+TASK_COMPLETED_MESSAGE: str
+PROCESSING_MESSAGE: str
+
+def create_step_response(agent: Agent, tool_calls: list[ToolCallInfo], is_done: bool, content: str, thinking_message: str | None) -> StreamingResponse:
+    """Compose the per-step payload emitted during browser-use streaming.
+
+    Args:
+        agent: Browser-use agent producing the step output.
+        tool_calls: Tool call descriptors extracted from the step.
+        is_done: Flag indicating whether this is the final step in the run.
+        content: High-level status text describing the step progress.
+        thinking_message: Optional preformatted thinking summary to attach.
+
+    Returns:
+        StreamingResponse: Serialized payload for the current streaming step.
+    """
+async def generate_thinking_message(content: str, tool_calls: list[dict[str, Any]], *, is_final: bool) -> str | None:
+    """Generate a user-facing thinking summary using the formatter LLM when available.
+
+    Args:
+        content: High-level status text describing the step progress.
+        tool_calls: Serialized tool call dictionaries with outputs.
+        is_final: Whether the task has completed.
+
+    Returns:
+        Markdown-formatted summary string, or ``None`` when not applicable.
+    """
+def create_error_response(error_message: str, recording_url: str = '') -> dict:
+    """Create a standardized error response.
+
+    Args:
+        error_message: The error message to include.
+        recording_url: The recording URL if available.
+
+    Returns:
+        dict: Standardized error response.
+    """
+def generate_step_content(tool_calls: list[ToolCallInfo], is_done: bool) -> str:
+    """Return user-friendly status text derived from tool call outputs.
+
+    Args:
+        tool_calls: Tool call descriptors extracted from the step.
+        is_done: Flag indicating whether this is the final step in the run.
+
+    Returns:
+        User-friendly status text string.
+    """
+def yield_iframe_activity(url: str, content: str) -> dict:
+    """Create and return an iframe activity streaming response.
+
+    Args:
+        url: The URL to display in the iframe.
+        content: The content message for the response.
+
+    Returns:
+        dict: Streaming response dictionary.
+    """
+def yield_status_message(content: str) -> dict:
+    """Create a status update event notifying clients about recovery attempts.
+
+    Args:
+        content: The status message content to include in the event.
+
+    Returns:
+        dict: Streaming response dictionary for the status update event.
+    """
+def yield_thinking_marker(marker_type: Literal['start', 'end']) -> dict:
+    """Create and return a thinking marker streaming response.
+
+    Args:
+        marker_type: Either 'start' or 'end' to indicate thinking phase.
+
+    Returns:
+        dict: Streaming response dictionary.
+    """

aip_agents/tools/browser_use/structured_data_parser.py

@@ -0,0 +1,257 @@
+"""High-level helpers for parsing and validating structured data extractor output.
+
+Authors:
+    Raymond Christopher (raymond.christopher@gdplabs.id)
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from aip_agents.tools.browser_use.structured_data_recovery import (
+    recover_concatenated_json_objects,
+    repair_json_blob,
+)
+from aip_agents.tools.browser_use.types import ToolCallInfo
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def detect_structured_data_failure(
+    tool_calls: Sequence[ToolCallInfo],
+    summarize_error: Callable[[str], str],
+) -> str | None:
+    """Return a descriptive error when structured data extraction yields no results.
+
+    Args:
+        tool_calls: Tool call descriptors extracted from the latest agent step.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Failure reason when extraction yielded nothing, otherwise None.
+    """
+    for call in tool_calls:
+        failure = structured_data_failure_for_call(call, summarize_error)
+        if failure:
+            return failure
+    return None
+
+
+def structured_data_failure_for_call(
+    call: ToolCallInfo,
+    summarize_error: Callable[[str], str],
+) -> str | None:
+    """Evaluate a single tool call for extractor failures.
+
+    Args:
+        call: Tool call descriptor encapsulating name/args/output.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Failure message when the call represents a bad extraction.
+    """
+    if call.name != "extract_structured_data":
+        return None
+
+    payload, parse_error = parse_structured_data(call.output or "", summarize_error)
+    if parse_error:
+        summary = summarize_error(parse_error)
+        return f"Structured data extraction emitted invalid JSON. Decoder reported: {summary}"
+    if payload is None:
+        return None
+
+    raw_output = call.output or ""
+    if isinstance(payload, dict):
+        error_message = payload_reports_error(payload, raw_output, summarize_error)
+        if error_message:
+            return error_message
+
+        empty_message = payload_reports_empty(payload, raw_output, summarize_error)
+        if empty_message:
+            return empty_message
+
+    return None
+
+
+def payload_reports_error(
+    payload: dict[str, Any],
+    raw_output: str,
+    summarize_error: Callable[[str], str],
+) -> str | None:
+    """Return a formatted error message when extractor status indicates failure.
+
+    Args:
+        payload: Parsed JSON payload emitted by the extractor.
+        raw_output: Original extractor output (used for fallback summaries).
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Human-readable failure string when extraction failed.
+    """
+    status = payload.get("status")
+    if isinstance(status, str) and status.lower() == "error":
+        message = payload.get("message") or payload.get("error") or raw_output
+        summary = summarize_error(message)
+        return f"Structured data extraction failed: {summary}"
+    return None
+
+
+def payload_reports_empty(
+    payload: dict[str, Any],
+    raw_output: str,
+    summarize_error: Callable[[str], str],
+) -> str | None:
+    """Return a formatted message when the extractor returned no usable data.
+
+    Args:
+        payload: Parsed JSON payload emitted by the extractor.
+        raw_output: Original extractor output string.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Human-readable failure message when no content was extracted.
+    """
+    products = payload.get("products")
+    count = payload.get("count")
+    available = payload.get("available")
+    products_found = payload.get("products_found")
+
+    no_products = isinstance(products, list) and len(products) == 0
+    explicit_zero = count == 0 or products_found == 0
+    explicitly_unavailable = available is False
+
+    if no_products or explicit_zero or explicitly_unavailable:
+        summary = summarize_error(raw_output)
+        return f"Structured data extraction returned no usable entries. Extractor response: {summary}"
+    return None
+
+
+def parse_structured_data(
+    output: str,
+    summarize_error: Callable[[str], str],
+) -> tuple[dict[str, Any] | list[Any] | None, str | None]:
+    """Extract the JSON blob emitted by extract_structured_data if present.
+
+    Args:
+        output: Raw string payload returned by extract_structured_data.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        tuple[dict[str, Any] | list[Any] | None, str | None]: Parsed JSON payload when extraction succeeds,
+            otherwise a tuple containing None and a diagnostic string on failure.
+    """
+    content = extract_content_after_marker(output)
+    if not content:
+        return None, None
+
+    json_blob = extract_json_blob(content)
+    if not json_blob:
+        return None, None
+
+    parsed_data = attempt_json_recovery(json_blob)
+    if parsed_data is not None:
+        return parsed_data, None
+
+    snippet = summarize_error(json_blob)
+    error_msg = "JSON parsing failed after all recovery attempts"
+    logger.warning(
+        "Structured data extractor emitted malformed JSON: %s. snippet=%s",
+        error_msg,
+        snippet,
+    )
+    return None, f"{error_msg}: {snippet}"
+
+
+def extract_content_after_marker(output: str) -> str | None:
+    """Extract content after the 'Extracted Content:' marker and clean trailing metadata.
+
+    Args:
+        output: Raw string payload returned by extract_structured_data.
+
+    Returns:
+        str | None: Cleaned content after the marker, or None if marker not found.
+    """
+    if "Extracted Content:" not in output:
+        return None
+
+    _, remainder = output.split("Extracted Content:", 1)
+
+    for marker in ("</extracted_content>", "<file_system>", "</file_system>"):
+        marker_index = remainder.find(marker)
+        if marker_index != -1:
+            remainder = remainder[:marker_index]
+            break
+
+    remainder = remainder.strip()
+    return remainder if remainder else None
+
+
+def extract_json_blob(content: str) -> str | None:
+    """Extract the JSON blob from content by finding delimiters and trimming trailing content.
+
+    Args:
+        content: Content string potentially containing JSON.
+
+    Returns:
+        str | None: Extracted JSON blob, or None if no valid JSON delimiters found.
+    """
+    first_bracket = content.find("[")
+    first_brace = content.find("{")
+    candidates = [index for index in (first_bracket, first_brace) if index != -1]
+    if not candidates:
+        return None
+
+    start = min(candidates)
+    json_blob = content[start:]
+
+    closing_char = "]" if json_blob.startswith("[") else "}"
+    end = json_blob.rfind(closing_char)
+    if end == -1:
+        return None
+    return json_blob[: end + 1].strip()
+
+
+def attempt_json_recovery(json_blob: str) -> dict[str, Any] | list[Any] | None:
+    """Attempt to parse JSON blob, with recovery strategies for common issues.
+
+    Args:
+        json_blob: JSON string to attempt parsing.
+
+    Returns:
+        dict[str, Any] | list[Any] | None: Parsed JSON data if successful, None if all recovery attempts fail.
+    """
+    try:
+        return json.loads(json_blob)
+    except json.JSONDecodeError as error:
+        recovered_payload = recover_concatenated_json_objects(json_blob)
+        if recovered_payload is not None:
+            return recovered_payload
+
+        repaired = repair_json_blob(json_blob)
+        if repaired:
+            try:
+                payload = json.loads(repaired)
+            except json.JSONDecodeError:
+                logger.debug("json_repair returned unrecoverable payload for structured data output.")
+            else:
+                logger.info(
+                    "Structured data extractor output repaired via json_repair and parsed successfully. original_error=%s",
+                    error.msg,
+                )
+                return payload
+    return None
+
+
+__all__ = [
+    "attempt_json_recovery",
+    "detect_structured_data_failure",
+    "extract_content_after_marker",
+    "extract_json_blob",
+    "parse_structured_data",
+    "payload_reports_empty",
+    "payload_reports_error",
+    "structured_data_failure_for_call",
+]

aip_agents/tools/browser_use/structured_data_parser.pyi

@@ -0,0 +1,86 @@
+from aip_agents.tools.browser_use.types import ToolCallInfo
+from collections.abc import Callable, Sequence
+from typing import Any
+
+__all__ = ['attempt_json_recovery', 'detect_structured_data_failure', 'extract_content_after_marker', 'extract_json_blob', 'parse_structured_data', 'payload_reports_empty', 'payload_reports_error', 'structured_data_failure_for_call']
+
+def detect_structured_data_failure(tool_calls: Sequence[ToolCallInfo], summarize_error: Callable[[str], str]) -> str | None:
+    """Return a descriptive error when structured data extraction yields no results.
+
+    Args:
+        tool_calls: Tool call descriptors extracted from the latest agent step.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Failure reason when extraction yielded nothing, otherwise None.
+    """
+def structured_data_failure_for_call(call: ToolCallInfo, summarize_error: Callable[[str], str]) -> str | None:
+    """Evaluate a single tool call for extractor failures.
+
+    Args:
+        call: Tool call descriptor encapsulating name/args/output.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Failure message when the call represents a bad extraction.
+    """
+def payload_reports_error(payload: dict[str, Any], raw_output: str, summarize_error: Callable[[str], str]) -> str | None:
+    """Return a formatted error message when extractor status indicates failure.
+
+    Args:
+        payload: Parsed JSON payload emitted by the extractor.
+        raw_output: Original extractor output (used for fallback summaries).
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Human-readable failure string when extraction failed.
+    """
+def payload_reports_empty(payload: dict[str, Any], raw_output: str, summarize_error: Callable[[str], str]) -> str | None:
+    """Return a formatted message when the extractor returned no usable data.
+
+    Args:
+        payload: Parsed JSON payload emitted by the extractor.
+        raw_output: Original extractor output string.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        str | None: Human-readable failure message when no content was extracted.
+    """
+def parse_structured_data(output: str, summarize_error: Callable[[str], str]) -> tuple[dict[str, Any] | list[Any] | None, str | None]:
+    """Extract the JSON blob emitted by extract_structured_data if present.
+
+    Args:
+        output: Raw string payload returned by extract_structured_data.
+        summarize_error: Function to summarize error messages.
+
+    Returns:
+        tuple[dict[str, Any] | list[Any] | None, str | None]: Parsed JSON payload when extraction succeeds,
+            otherwise a tuple containing None and a diagnostic string on failure.
+    """
+def extract_content_after_marker(output: str) -> str | None:
+    """Extract content after the 'Extracted Content:' marker and clean trailing metadata.
+
+    Args:
+        output: Raw string payload returned by extract_structured_data.
+
+    Returns:
+        str | None: Cleaned content after the marker, or None if marker not found.
+    """
+def extract_json_blob(content: str) -> str | None:
+    """Extract the JSON blob from content by finding delimiters and trimming trailing content.
+
+    Args:
+        content: Content string potentially containing JSON.
+
+    Returns:
+        str | None: Extracted JSON blob, or None if no valid JSON delimiters found.
+    """
+def attempt_json_recovery(json_blob: str) -> dict[str, Any] | list[Any] | None:
+    """Attempt to parse JSON blob, with recovery strategies for common issues.
+
+    Args:
+        json_blob: JSON string to attempt parsing.
+
+    Returns:
+        dict[str, Any] | list[Any] | None: Parsed JSON data if successful, None if all recovery attempts fail.
+    """

aip_agents/tools/browser_use/structured_data_recovery.py

@@ -0,0 +1,204 @@
+"""Utilities for recovering malformed structured-data payloads emitted by browser-use.
+
+Authors:
+    Raymond Christopher (raymond.christopher@gdplabs.id)
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from json_repair import repair_json
+
+from aip_agents.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def recover_concatenated_json_objects(json_blob: str) -> dict[str, Any] | None:
+    """Normalize concatenated JSON object strings into a structured payload.
+
+    Args:
+        json_blob: Raw JSON-like string returned by the structured data extractor.
+
+    Returns:
+        dict[str, Any] | None: Standardized payload when multiple objects are recovered,
+        otherwise None.
+    """
+    segments = _split_top_level_json_objects(json_blob)
+    if len(segments) <= 1:
+        return None
+
+    try:
+        records = [json.loads(segment) for segment in segments]
+    except json.JSONDecodeError:
+        return None
+
+    count = len(records)
+    logger.info("Structured data extractor returned concatenated JSON objects. recovered=%s", count)
+    return {
+        "status": "ok",
+        "items": records,
+        "products": records,
+        "count": count,
+        "products_found": count,
+        "available": bool(records),
+    }
+
+
+def repair_json_blob(json_blob: str) -> str | None:
+    """Apply json_repair to malformed JSON strings and return the mutated payload.
+
+    Args:
+        json_blob: Raw JSON string that may contain syntax mistakes.
+
+    Returns:
+        str | None: Repaired JSON string when modifications were applied, otherwise None.
+    """
+    try:
+        repaired = repair_json(json_blob)
+    except Exception:
+        logger.debug("json_repair failed to repair structured data output.", exc_info=True)
+        return None
+
+    if not repaired or repaired == json_blob:
+        return None
+
+    return repaired
+
+
+def _split_top_level_json_objects(json_blob: str) -> list[str]:
+    """Split concatenated JSON objects while respecting string literals.
+
+    Args:
+        json_blob: The JSON string containing concatenated objects to split.
+    """
+    splitter = _JsonObjectSplitter(json_blob)
+    return splitter.split_objects()
+
+
+def _has_only_separators(value: str) -> bool:
+    """Return True when the substring only contains whitespace or commas.
+
+    Args:
+        value: The string to check for separators only.
+    """
+    return value.strip(" \t\r\n,") == ""
+
+
+class _JsonObjectSplitter:
+    """Helper class to split JSON objects with reduced cognitive complexity."""
+
+    def __init__(self, json_blob: str) -> None:
+        """Initialize the splitter with the raw JSON string.
+
+        Args:
+            json_blob: Raw string potentially containing concatenated JSON objects.
+        """
+        self.json_blob = json_blob
+        self.segments: list[str] = []
+        self.depth = 0
+        self.start: int | None = None
+        self.last_end = 0
+        self.in_string = False
+        self.escaping = False
+
+    def split_objects(self) -> list[str]:
+        """Main method to split JSON objects."""
+        if not self._parse_characters():
+            return []
+
+        if not self._validate_final_state():
+            return []
+
+        return self.segments
+
+    def _parse_characters(self) -> bool:
+        """Parse each character and build segments. Returns False if invalid."""
+        for index, char in enumerate(self.json_blob):
+            if not self._process_character(char, index):
+                return False
+        return True
+
+    def _process_character(self, char: str, index: int) -> bool:
+        """Process a single character. Returns False if parsing should stop.
+
+        Args:
+            char: The character to process.
+            index: The position of the character in the JSON blob.
+
+        Returns:
+            True if processing should continue, False if parsing should stop.
+        """
+        if self.escaping:
+            self.escaping = False
+            return True
+
+        if char == "\\":
+            self.escaping = True
+            return True
+
+        if char == '"':
+            self.in_string = not self.in_string
+            return True
+
+        if self.in_string:
+            return True
+
+        return self._process_non_string_char(char, index)
+
+    def _process_non_string_char(self, char: str, index: int) -> bool:
+        """Process characters outside of strings.
+
+        Args:
+            char: The character to process (not in a string).
+            index: The position of the character in the JSON blob.
+
+        Returns:
+            True if processing should continue, False if invalid structure found.
+        """
+        if char == "{":
+            return self._handle_open_brace(index)
+        if char == "}":
+            return self._handle_close_brace(index)
+        return True
+
+    def _handle_open_brace(self, index: int) -> bool:
+        """Handle opening brace character.
+
+        Args:
+            index: The position of the opening brace in the JSON blob.
+
+        Returns:
+            True if brace should be processed, False if invalid separators found.
+        """
+        if self.depth == 0:
+            if not _has_only_separators(self.json_blob[self.last_end : index]):
+                return False
+            self.start = index
+        self.depth += 1
+        return True
+
+    def _handle_close_brace(self, index: int) -> bool:
+        """Handle closing brace character.
+
+        Args:
+            index: The position of the closing brace in the JSON blob.
+
+        Returns:
+            Always True as closing braces don't cause parsing failures.
+        """
+        self.depth -= 1
+        if self.depth == 0 and self.start is not None:
+            end = index + 1
+            self.segments.append(self.json_blob[self.start : end])
+            self.last_end = end
+        return True
+
+    def _validate_final_state(self) -> bool:
+        """Validate that parsing completed successfully."""
+        if self.depth != 0 or self.in_string:
+            return False
+
+        return _has_only_separators(self.json_blob[self.last_end :])

aip_agents/tools/browser_use/structured_data_recovery.pyi

@@ -0,0 +1,43 @@
+from _typeshed import Incomplete
+from aip_agents.utils.logger import get_logger as get_logger
+from typing import Any
+
+logger: Incomplete
+
+def recover_concatenated_json_objects(json_blob: str) -> dict[str, Any] | None:
+    """Normalize concatenated JSON object strings into a structured payload.
+
+    Args:
+        json_blob: Raw JSON-like string returned by the structured data extractor.
+
+    Returns:
+        dict[str, Any] | None: Standardized payload when multiple objects are recovered,
+        otherwise None.
+    """
+def repair_json_blob(json_blob: str) -> str | None:
+    """Apply json_repair to malformed JSON strings and return the mutated payload.
+
+    Args:
+        json_blob: Raw JSON string that may contain syntax mistakes.
+
+    Returns:
+        str | None: Repaired JSON string when modifications were applied, otherwise None.
+    """
+
+class _JsonObjectSplitter:
+    """Helper class to split JSON objects with reduced cognitive complexity."""
+    json_blob: Incomplete
+    segments: list[str]
+    depth: int
+    start: int | None
+    last_end: int
+    in_string: bool
+    escaping: bool
+    def __init__(self, json_blob: str) -> None:
+        """Initialize the splitter with the raw JSON string.
+
+        Args:
+            json_blob: Raw string potentially containing concatenated JSON objects.
+        """
+    def split_objects(self) -> list[str]:
+        """Main method to split JSON objects."""