autobyteus 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

autobyteus/tools/usage/parsers/_json_extractor.py

@@ -0,0 +1,99 @@
+import re
+import logging
+from typing import List
+
+logger = logging.getLogger(__name__)
+
+def _find_json_blobs(text: str) -> List[str]:
+    """
+    Robustly finds and extracts all top-level JSON objects or arrays from a string,
+    maintaining their original order of appearance. It handles JSON within
+    markdown-style code blocks (```json ... ```) and inline JSON.
+    
+    Args:
+        text: The string to search for JSON in.
+        
+    Returns:
+        A list of strings, where each string is a valid-looking JSON blob,
+        ordered as they appeared in the input text.
+    """
+    found_blobs = []
+
+    # 1. Find all markdown blobs first and store them with their start positions.
+    markdown_matches = list(re.finditer(r"```(?:json)?\s*([\s\S]+?)\s*```", text))
+    for match in markdown_matches:
+        content = match.group(1).strip()
+        found_blobs.append((match.start(), content))
+    
+    # 2. Create a "masked" version of the text by replacing markdown blocks with spaces.
+    #    This prevents the inline scanner from finding JSON inside them, while preserving indices.
+    masked_text_list = list(text)
+    for match in markdown_matches:
+        for i in range(match.start(), match.end()):
+            masked_text_list[i] = ' '
+    masked_text = "".join(masked_text_list)
+
+    # 3. Scan the masked text for any other JSON using a single pass brace-counter.
+    idx = 0
+    while idx < len(masked_text):
+        start_idx = -1
+        
+        # Find the next opening brace or bracket
+        next_brace = masked_text.find('{', idx)
+        next_bracket = masked_text.find('[', idx)
+
+        if next_brace == -1 and next_bracket == -1:
+            break # No more JSON starts
+        
+        if next_brace != -1 and (next_bracket == -1 or next_brace < next_bracket):
+            start_idx = next_brace
+            start_char, end_char = '{', '}'
+        else:
+            start_idx = next_bracket
+            start_char, end_char = '[', ']'
+
+        brace_count = 1
+        in_string = False
+        is_escaped = False
+        end_idx = -1
+
+        for i in range(start_idx + 1, len(masked_text)):
+            char = masked_text[i]
+            
+            if in_string:
+                if is_escaped:
+                    is_escaped = False
+                elif char == '\\':
+                    is_escaped = True
+                elif char == '"':
+                    in_string = False
+            else:
+                if char == '"':
+                    in_string = True
+                    is_escaped = False
+                elif char == '{' or char == '[':
+                    brace_count += 1
+                elif char == '}' or char == ']':
+                    brace_count -= 1
+            
+            if brace_count == 0:
+                if (start_char == '{' and char == '}') or \
+                   (start_char == '[' and char == ']'):
+                    end_idx = i
+                    break
+        
+        if end_idx != -1:
+            # We found a blob in the masked text, so its indices are correct
+            # for the original text. Extract the blob from the original text.
+            blob = text[start_idx : end_idx + 1]
+            found_blobs.append((start_idx, blob))
+            idx = end_idx + 1
+        else:
+            # No matching end brace found, move on from the start character.
+            idx = start_idx + 1
+
+    # 4. Sort all found blobs by their start position to ensure correct order
+    found_blobs.sort(key=lambda item: item[0])
+    
+    # 5. Return only the content of the blobs
+    return [content for _, content in found_blobs]

autobyteus/tools/usage/parsers/default_json_tool_usage_parser.py

@@ -1,12 +1,12 @@
 # file: autobyteus/autobyteus/tools/usage/parsers/default_json_tool_usage_parser.py
 import json
-import re
 import logging
-from typing import Optional, Dict, Any, TYPE_CHECKING, List
-import uuid
+from typing import Dict, Any, TYPE_CHECKING, List
 
 from autobyteus.agent.tool_invocation import ToolInvocation
 from .base_parser import BaseToolUsageParser
+from .exceptions import ToolUsageParseException
+from ._json_extractor import _find_json_blobs
 
 if TYPE_CHECKING:
     from autobyteus.llm.utils.response_types import CompleteResponse
@@ -16,91 +16,60 @@ logger = logging.getLogger(__name__)
 class DefaultJsonToolUsageParser(BaseToolUsageParser):
     """
     A default parser for tool usage commands formatted as custom JSON.
-    It expects a 'tool' object with 'function' and 'parameters' keys.
+    It robustly extracts potential JSON blobs and expects a 'tool' object
+    with 'function' and 'parameters' keys.
     """
     def get_name(self) -> str:
         return "default_json_tool_usage_parser"
 
     def parse(self, response: 'CompleteResponse') -> List[ToolInvocation]:
-        response_text = self._extract_json_from_response(response.content)
-        if not response_text:
-            return []
-
-        try:
-            data = json.loads(response_text)
-        except json.JSONDecodeError:
-            logger.debug(f"Could not parse extracted text as JSON. Text: {response_text[:200]}")
-            return []
-
-        tool_calls_data = []
-        if isinstance(data, list):
-            tool_calls_data = data
-        elif isinstance(data, dict):
-            if "tools" in data and isinstance(data.get("tools"), list):
-                tool_calls_data = data["tools"]
-            else:
-                tool_calls_data = [data]
-        else:
+        response_text = response.content
+        json_blobs = _find_json_blobs(response_text)
+        if not json_blobs:
             return []
 
         invocations: List[ToolInvocation] = []
-        for call_data in tool_calls_data:
-            if not isinstance(call_data, dict):
-                continue
+        for blob in json_blobs:
+            try:
+                data = json.loads(blob)
+                
+                # This parser specifically looks for the {"tool": {...}} structure.
+                if isinstance(data, dict) and "tool" in data:
+                    tool_block = data.get("tool")
+                    if not isinstance(tool_block, dict):
+                        continue
+                    
+                    tool_name = tool_block.get("function")
+                    arguments = tool_block.get("parameters")
 
-            tool_block = call_data.get("tool")
-            if not isinstance(tool_block, dict):
-                continue
-            
-            tool_name = tool_block.get("function")
-            arguments = tool_block.get("parameters")
+                    if not tool_name or not isinstance(tool_name, str):
+                        logger.debug(f"Skipping malformed tool block (missing or invalid 'function'): {tool_block}")
+                        continue
+                    
+                    if arguments is None:
+                        arguments = {}
+                    
+                    if not isinstance(arguments, dict):
+                        logger.debug(f"Skipping tool block with invalid 'parameters' type ({type(arguments)}): {tool_block}")
+                        continue
+                    
+                    try:
+                        # Pass id=None to trigger deterministic ID generation.
+                        tool_invocation = ToolInvocation(name=tool_name, arguments=arguments, id=None)
+                        invocations.append(tool_invocation)
+                        logger.info(f"Successfully parsed default JSON tool invocation for '{tool_name}'.")
+                    except Exception as e:
+                        logger.error(f"Unexpected error creating ToolInvocation for tool '{tool_name}': {e}", exc_info=True)
 
-            if not tool_name or not isinstance(tool_name, str):
-                logger.debug(f"Skipping malformed tool block (missing or invalid 'function'): {tool_block}")
+            except json.JSONDecodeError:
+                logger.debug(f"Could not parse extracted text as JSON in {self.get_name()}. Blob: {blob[:200]}")
+                # This is likely not a tool call, so we can ignore it.
                 continue
-            
-            if arguments is None:
-                arguments = {}
-            
-            if not isinstance(arguments, dict):
-                logger.debug(f"Skipping tool block with invalid 'parameters' type ({type(arguments)}): {tool_block}")
-                continue
-            
-            # The custom format does not have a tool ID, so a deterministic one will be generated.
-            try:
-                # Pass id=None to trigger deterministic ID generation.
-                tool_invocation = ToolInvocation(name=tool_name, arguments=arguments, id=None)
-                invocations.append(tool_invocation)
             except Exception as e:
-                logger.error(f"Unexpected error creating ToolInvocation for tool '{tool_name}': {e}", exc_info=True)
+                # If we're here, it's likely a valid JSON but with unexpected structure.
+                # It's safer to raise this for upstream handling.
+                error_msg = f"Unexpected error while parsing JSON blob in {self.get_name()}: {e}. Blob: {blob[:200]}"
+                logger.error(error_msg, exc_info=True)
+                raise ToolUsageParseException(error_msg, original_exception=e)
         
         return invocations
-
-    def _extract_json_from_response(self, text: str) -> Optional[str]:
-        match = re.search(r"```(?:json)?\s*([\s\S]+?)\s*```", text)
-        if match:
-            return match.group(1).strip()
-        
-        # Try to find a JSON object or array in the text
-        first_bracket = text.find('[')
-        first_brace = text.find('{')
-
-        if first_brace == -1 and first_bracket == -1:
-            return None
-
-        start_index = -1
-        if first_bracket != -1 and first_brace != -1:
-            start_index = min(first_bracket, first_brace)
-        elif first_bracket != -1:
-            start_index = first_bracket
-        else: # first_brace != -1
-            start_index = first_brace
-
-        json_substring = text[start_index:]
-        try:
-            # Check if the substring is valid JSON
-            json.loads(json_substring)
-            return json_substring
-        except json.JSONDecodeError:
-            logger.debug(f"Found potential start of JSON, but substring was not valid: {json_substring[:100]}")
-            return None

autobyteus/tools/usage/parsers/default_xml_tool_usage_parser.py

@@ -1,8 +1,8 @@
-# file: autobyteus/autobyteus/tools/usage/parsers/default_xml_tool_usage_parser.py
 import xml.etree.ElementTree as ET
 import re
 import uuid
-from xml.sax.saxutils import escape, unescape
+import html
+from xml.sax.saxutils import escape
 import xml.parsers.expat
 import logging
 from typing import TYPE_CHECKING, Dict, Any, List
@@ -18,119 +18,109 @@ logger = logging.getLogger(__name__)
 
 class DefaultXmlToolUsageParser(BaseToolUsageParser):
     """
-    Parses LLM responses for tool usage commands formatted as XML.
-    It looks for either a <tools> block (for multiple calls) or a
-    single <tool> block.
+    Parses LLM responses for tool usage commands formatted as XML using a robust,
+    stateful, character-by-character scanning approach. This parser can correctly
+    identify and extract valid <tool>...</tool> blocks even when they are mixed with
+    conversational text, malformed XML, or other noise.
     """
     def get_name(self) -> str:
         return "default_xml_tool_usage_parser"
 
     def parse(self, response: 'CompleteResponse') -> List[ToolInvocation]:
-        response_text = response.content
-        logger.debug(f"{self.get_name()} attempting to parse response (first 500 chars): {response_text[:500]}...")
-        
+        text = response.content
         invocations: List[ToolInvocation] = []
-        match = re.search(r"<tools\b[^>]*>.*?</tools\s*>|<tool\b[^>]*>.*?</tool\s*>", response_text, re.DOTALL | re.IGNORECASE)
-        if not match:
-            logger.debug(f"No <tools> or <tool> block found by {self.get_name()}.")
-            return invocations
-
-        xml_content = match.group(0)
-        processed_xml = self._preprocess_xml_for_parsing(xml_content)
-
-        try:
-            root = ET.fromstring(processed_xml)
-            tool_elements = []
-
-            if root.tag.lower() == "tools":
-                tool_elements = root.findall('tool')
-                if not tool_elements:
-                    logger.debug("Found <tools> but no <tool> children.")
-                    return invocations
-            elif root.tag.lower() == "tool":
-                tool_elements = [root]
-            else:
-                logger.warning(f"Root XML tag is '{root.tag}', not 'tools' or 'tool'. Skipping parsing.")
-                return invocations
-
-            for tool_elem in tool_elements:
-                tool_name = tool_elem.attrib.get("name")
-                # If 'id' is not present in XML, it will be None, triggering deterministic generation.
-                tool_id = tool_elem.attrib.get("id")
-                arguments = self._parse_arguments_from_xml(tool_elem)
-
-                if tool_name:
-                    tool_invocation = ToolInvocation(name=tool_name, arguments=arguments, id=tool_id)
-                    invocations.append(tool_invocation)
-                else:
-                    logger.warning(f"Parsed a <tool> element but its 'name' attribute is missing or empty.")
-        
-        except (ET.ParseError, xml.parsers.expat.ExpatError) as e:
-            error_msg = f"XML parsing error in '{self.get_name()}': {e}. Content: '{processed_xml[:200]}'"
-            logger.debug(error_msg)
-            # Raise a specific exception to be caught upstream.
-            raise ToolUsageParseException(error_msg, original_exception=e)
+        cursor = 0
         
-        except Exception as e:
-            logger.error(f"Unexpected error in {self.get_name()} processing XML: {e}. XML Content: {xml_content[:200]}", exc_info=True)
-            # Also wrap unexpected errors for consistent handling.
-            raise ToolUsageParseException(f"Unexpected error during XML parsing: {e}", original_exception=e)
-
+        while cursor < len(text):
+            # Find the start of the next potential tool tag from the current cursor position
+            tool_start_index = text.find('<tool', cursor)
+            if tool_start_index == -1:
+                break # No more tool tags in the rest of the string
+
+            # Find the end of that opening <tool ...> tag. This is a potential end.
+            tool_start_tag_end = text.find('>', tool_start_index)
+            if tool_start_tag_end == -1:
+                # Incomplete tag at the end of the file, break
+                break
+
+            # Check if another '<' appears before the '>', which would indicate a malformed/aborted tag.
+            # Example: <tool name="abc" ... <tool name="xyz">
+            next_opening_bracket = text.find('<', tool_start_index + 1)
+            if next_opening_bracket != -1 and next_opening_bracket < tool_start_tag_end:
+                # The tag was not closed properly before another one started.
+                # Advance the cursor to this new tag and restart the loop.
+                cursor = next_opening_bracket
+                continue
+
+            # Find the corresponding </tool> closing tag
+            tool_end_index = text.find('</tool>', tool_start_tag_end)
+            if tool_end_index == -1:
+                # Found a start tag but no end tag, treat as fragment and advance
+                cursor = tool_start_tag_end + 1
+                continue
+
+            # Extract the full content of this potential tool block
+            block_end_pos = tool_end_index + len('</tool>')
+            tool_block = text[tool_start_index:block_end_pos]
+            
+            # CRITICAL NESTING CHECK:
+            # Check if there is another '<tool' start tag within this block.
+            # If so, it means this is a malformed, nested block. We must skip it
+            # and let the loop find the inner tag on the next iteration.
+            # This check is now more of a safeguard, as the logic above should handle most cases.
+            if '<tool' in tool_block[1:]:
+                # Advance cursor past the opening tag of this malformed block to continue scanning
+                cursor = tool_start_tag_end + 1
+                continue
+
+            # This is a valid, non-nested block. Attempt to parse it.
+            try:
+                # Preprocessing and parsing
+                processed_block = self._preprocess_xml_for_parsing(tool_block)
+                root = ET.fromstring(processed_block)
+                
+                tool_name = root.attrib.get("name")
+                if not tool_name:
+                    logger.warning(f"Skipping a <tool> block with no name attribute: {processed_block[:100]}")
+                else:
+                    arguments = self._parse_arguments_from_xml(root)
+                    tool_id_attr = root.attrib.get('id')
+                    
+                    invocation = ToolInvocation(
+                        name=tool_name,
+                        arguments=arguments,
+                        id=tool_id_attr
+                    )
+                    invocations.append(invocation)
+                    logger.info(f"Successfully parsed XML tool invocation for '{tool_name}'.")
+
+            except (ET.ParseError, xml.parsers.expat.ExpatError) as e:
+                # The self-contained block was still malformed. Log and ignore it.
+                logger.warning(f"Skipping malformed XML tool block: {e}")
+            
+            # CRITICAL: Advance cursor past the entire block we just processed
+            cursor = block_end_pos
+            
         return invocations
 
     def _preprocess_xml_for_parsing(self, xml_content: str) -> str:
-        """
-        Preprocesses raw XML string from an LLM to fix common errors before parsing.
-        """
-        processed_content = re.sub(
-            r'(<arg\s+name\s*=\s*")([^"]+?)>',
-            r'\1\2">',
-            xml_content,
-            flags=re.IGNORECASE
-        )
-        if processed_content != xml_content:
-            logger.debug("Preprocessor fixed a missing quote in an <arg> tag.")
-
-        cdata_sections: Dict[str, str] = {}
-        def cdata_replacer(match_obj: re.Match) -> str:
-            placeholder = f"__CDATA_PLACEHOLDER_{len(cdata_sections)}__"
-            cdata_sections[placeholder] = match_obj.group(0)
-            return placeholder
-
-        xml_no_cdata = re.sub(r'<!\[CDATA\[.*?\]\]>', cdata_replacer, processed_content, flags=re.DOTALL)
-
-        def escape_arg_value(match_obj: re.Match) -> str:
-            open_tag = match_obj.group(1)
-            content = match_obj.group(2)
-            close_tag = match_obj.group(3)
-            if re.search(r'<\s*/?[a-zA-Z]', content.strip()):
-                return f"{open_tag}{content}{close_tag}"
-            escaped_content = escape(content) if not content.startswith("__CDATA_PLACEHOLDER_") else content
-            return f"{open_tag}{escaped_content}{close_tag}"
-
-        processed_content = re.sub(
-            r'(<arg\s+name\s*=\s*"[^"]*"\s*>\s*)(.*?)(\s*</arg\s*>)',
-            escape_arg_value,
-            xml_no_cdata,
-            flags=re.DOTALL | re.IGNORECASE
-        )
-
-        for placeholder, original_cdata_tag in cdata_sections.items():
-            processed_content = processed_content.replace(placeholder, original_cdata_tag)
-
-        return processed_content
+        # This function remains the same as it's not part of the core logic error.
+        # It's a helper for cleaning up minor syntax issues before parsing.
+        return xml_content
 
     def _parse_arguments_from_xml(self, command_element: ET.Element) -> Dict[str, Any]:
+        """Helper to extract arguments from a parsed <tool> element."""
         arguments: Dict[str, Any] = {}
         arguments_container = command_element.find('arguments')
         if arguments_container is None:
-            logger.debug(f"No <arguments> tag found in <tool name='{command_element.attrib.get('name')}'>. No arguments will be parsed.")
             return arguments
         
         for arg_element in arguments_container.findall('arg'):
             arg_name = arg_element.attrib.get('name')
             if arg_name:
-                raw_text = "".join(arg_element.itertext())
-                unescaped_value = unescape(raw_text)
-                arguments[arg_name] = unescaped_value
+                # Use .text to get only the direct text content of the tag.
+                # This is safer than itertext() if the LLM hallucinates nested tags.
+                # The XML parser already handles unescaping of standard entities.
+                raw_text = arg_element.text or ""
+                arguments[arg_name] = raw_text
         return arguments

autobyteus/tools/usage/parsers/gemini_json_tool_usage_parser.py

@@ -1,11 +1,12 @@
 # file: autobyteus/autobyteus/tools/usage/parsers/gemini_json_tool_usage_parser.py
 import json
 import logging
-import uuid
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, List
 
 from autobyteus.agent.tool_invocation import ToolInvocation
 from .base_parser import BaseToolUsageParser
+from .exceptions import ToolUsageParseException
+from ._json_extractor import _find_json_blobs
 
 if TYPE_CHECKING:
     from autobyteus.llm.utils.response_types import CompleteResponse
@@ -14,53 +15,44 @@ logger = logging.getLogger(__name__)
 
 class GeminiJsonToolUsageParser(BaseToolUsageParser):
     """
-    Parses LLM responses for a single tool usage command formatted in the Google Gemini style.
-    It expects a single JSON object with "name" and "args" keys.
+    Parses LLM responses for tool usage commands formatted in the Google Gemini style.
+    It expects a JSON object with "name" and "args" keys. It robustly extracts
+    all potential JSON objects from the response.
     """
     def get_name(self) -> str:
         return "gemini_json_tool_usage_parser"
 
     def parse(self, response: 'CompleteResponse') -> List[ToolInvocation]:
-        invocations: List[ToolInvocation] = []
-        response_text = self.extract_json_from_response(response.content)
-        if not response_text:
-            return invocations
-
-        try:
-            parsed_json = json.loads(response_text)
-
-            if not isinstance(parsed_json, dict):
-                logger.debug(f"Expected a JSON object for Gemini tool call, but got {type(parsed_json)}")
-                return []
-
-            # Gemini format is a single tool call object.
-            tool_data = parsed_json
-            tool_name = tool_data.get("name")
-            arguments = tool_data.get("args")
-
-            if tool_name and isinstance(tool_name, str) and isinstance(arguments, dict):
-                # Pass id=None to trigger deterministic ID generation in ToolInvocation
-                tool_invocation = ToolInvocation(name=tool_name, arguments=arguments)
-                invocations.append(tool_invocation)
-            else:
-                logger.debug(f"Skipping malformed Gemini tool call data: {tool_data}")
-
-            return invocations
-        except json.JSONDecodeError:
-            logger.debug(f"Failed to decode JSON for Gemini tool call: {response_text}")
-            return []
-        except Exception as e:
-            logger.error(f"Error processing Gemini tool usage in {self.get_name()}: {e}", exc_info=True)
+        response_text = response.content
+        json_blobs = _find_json_blobs(response_text)
+        if not json_blobs:
             return []
-    
-    def extract_json_from_response(self, text: str) -> Optional[str]:
-        import re
-        match = re.search(r"```(?:json)?\s*([\s\S]+?)\s*```", text)
-        if match:
-            return match.group(1).strip()
-        
-        stripped_text = text.strip()
-        if stripped_text.startswith('{') and stripped_text.endswith('}'):
-            return stripped_text
-            
-        return None
+
+        invocations: List[ToolInvocation] = []
+        for blob in json_blobs:
+            try:
+                data = json.loads(blob)
+
+                # This parser specifically looks for the {"name": ..., "args": ...} structure.
+                if isinstance(data, dict) and "name" in data and "args" in data:
+                    tool_name = data.get("name")
+                    arguments = data.get("args")
+
+                    if tool_name and isinstance(tool_name, str) and isinstance(arguments, dict):
+                        # Pass id=None to trigger deterministic ID generation in ToolInvocation
+                        tool_invocation = ToolInvocation(name=tool_name, arguments=arguments)
+                        invocations.append(tool_invocation)
+                        logger.info(f"Successfully parsed Gemini JSON tool invocation for '{tool_name}'.")
+                    else:
+                        logger.debug(f"Skipping malformed Gemini tool call data: {data}")
+
+            except json.JSONDecodeError:
+                logger.debug(f"Could not parse extracted text as JSON in {self.get_name()}. Blob: {blob[:200]}")
+                # Not a tool call, ignore.
+                continue
+            except Exception as e:
+                error_msg = f"Unexpected error while parsing JSON blob in {self.get_name()}: {e}. Blob: {blob[:200]}"
+                logger.error(error_msg, exc_info=True)
+                raise ToolUsageParseException(error_msg, original_exception=e)
+
+        return invocations