autobyteus 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

autobyteus/agent/streaming/parser/states/delimited_content_state.py

@@ -0,0 +1,146 @@
+"""
+DelimitedContentState: Base state for content blocks terminated by a closing tag.
+
+Provides streaming-safe emission with a holdback tail to prevent partial
+closing tags from being emitted.
+"""
+from typing import TYPE_CHECKING, Optional, Dict
+
+from .base_state import BaseState
+from ..events import SegmentType
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class DelimitedContentState(BaseState):
+    """
+    Base class for parsing delimited content blocks.
+
+    Subclasses must define:
+    - CLOSING_TAG: The closing tag string (e.g., "</file>")
+    - SEGMENT_TYPE: SegmentType for emitted segments
+    """
+
+    CLOSING_TAG = ""
+    SEGMENT_TYPE: Optional[SegmentType] = None
+
+    def __init__(
+        self,
+        context: "ParserContext",
+        opening_tag: str,
+        closing_tag_override: Optional[str] = None,
+    ):
+        super().__init__(context)
+        self._opening_tag = opening_tag
+        self._segment_started = False
+        self._tail = ""
+        self._closing_tag = closing_tag_override if closing_tag_override is not None else self.CLOSING_TAG
+        self._closing_tag_lower = self._closing_tag.lower()
+        self._holdback_len = max(len(self._closing_tag) - 1, 0)
+
+    def _can_start_segment(self) -> bool:
+        """Return False to emit opening tag as text and abort."""
+        return True
+
+    def _get_start_metadata(self) -> Dict[str, str]:
+        """Metadata to include on segment start."""
+        return {}
+
+    def _opening_content(self) -> Optional[str]:
+        """Optional content to emit immediately after segment start."""
+        return None
+
+    def _on_segment_complete(self) -> None:
+        """Hook invoked just before segment end is emitted."""
+        return None
+
+    def _should_emit_closing_tag(self) -> bool:
+        """Whether to emit the closing tag as content."""
+        return False
+
+    def run(self) -> None:
+        from .text_state import TextState
+
+        if not self._segment_started:
+            if not self._can_start_segment():
+                self.context.append_text_segment(self._opening_tag)
+                self.context.transition_to(TextState(self.context))
+                return
+
+            if self.SEGMENT_TYPE is None:
+                raise RuntimeError("SEGMENT_TYPE is not defined for DelimitedContentState.")
+
+            self.context.emit_segment_start(self.SEGMENT_TYPE, **self._get_start_metadata())
+            self._segment_started = True
+
+            opening_content = self._opening_content()
+            if opening_content:
+                self.context.emit_segment_content(opening_content)
+
+        if not self.context.has_more_chars():
+            return
+
+        available = self.context.consume_remaining()
+        combined = self._tail + available
+
+        if combined:
+            idx = combined.lower().find(self._closing_tag_lower)
+        else:
+            idx = -1
+
+        if idx != -1:
+            content_before = combined[:idx]
+            if content_before:
+                self.context.emit_segment_content(content_before)
+
+            if self._should_emit_closing_tag() and self._closing_tag:
+                self.context.emit_segment_content(self._closing_tag)
+
+            tail_len = len(self._tail)
+            closing_len = len(self._closing_tag)
+            if idx < tail_len:
+                consumed_from_available = idx + closing_len - tail_len
+            else:
+                consumed_from_available = (idx - tail_len) + closing_len
+
+            extra = len(available) - consumed_from_available
+            if extra > 0:
+                self.context.rewind_by(extra)
+
+            self._tail = ""
+            self._on_segment_complete()
+            self.context.emit_segment_end()
+            self.context.transition_to(TextState(self.context))
+            return
+
+        if self._holdback_len == 0:
+            if combined:
+                self.context.emit_segment_content(combined)
+            self._tail = ""
+            return
+
+        if len(combined) > self._holdback_len:
+            safe = combined[:-self._holdback_len]
+            if safe:
+                self.context.emit_segment_content(safe)
+            self._tail = combined[-self._holdback_len:]
+        else:
+            self._tail = combined
+
+    def finalize(self) -> None:
+        from .text_state import TextState
+
+        remaining = self.context.consume_remaining() if self.context.has_more_chars() else ""
+
+        if not self._segment_started:
+            text = self._opening_tag + self._tail + remaining
+            if text:
+                self.context.append_text_segment(text)
+        else:
+            if self._tail or remaining:
+                self.context.emit_segment_content(self._tail + remaining)
+            self._tail = ""
+            self.context.emit_segment_end()
+
+        self.context.transition_to(TextState(self.context))

autobyteus/agent/streaming/parser/states/json_initialization_state.py

@@ -0,0 +1,144 @@
+"""
+JsonInitializationState: Analyzes potential JSON tool calls after a '{' or '[' is detected.
+
+This state buffers characters to determine if a JSON structure is a tool call
+based on a signature check strategy. Used for providers like OpenAI that use
+JSON format instead of XML.
+"""
+from typing import TYPE_CHECKING, Optional, List
+
+from .base_state import BaseState
+from ..events import SegmentType
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class JsonToolSignatureChecker:
+    """
+    Checks if a JSON buffer matches known tool call signatures.
+    
+    Common JSON tool call formats:
+    - OpenAI: {"name": "tool_name", "arguments": {...}}
+    - Anthropic: Similar structure
+    
+    Returns:
+    - 'match': Buffer matches a tool signature
+    - 'partial': Buffer could still match (keep buffering)
+    - 'no_match': Buffer definitely not a tool call
+    """
+    
+    def __init__(self, patterns: Optional[List[str]] = None):
+        """
+        Initialize with custom patterns or defaults.
+        
+        Args:
+            patterns: List of JSON prefixes that indicate tool calls.
+                     Uses defaults if not provided.
+        """
+        from ..parser_context import ParserConfig
+        self._patterns = patterns or ParserConfig.DEFAULT_JSON_PATTERNS
+    
+    def check_signature(self, buffer: str) -> str:
+        """
+        Check if the buffer matches a tool call signature.
+        
+        Returns 'match', 'partial', or 'no_match'.
+        """
+        # Normalize whitespace for checking
+        normalized = buffer.replace(" ", "").replace("\n", "").replace("\t", "")
+        
+        for pattern in self._patterns:
+            normalized_pattern = pattern.replace(" ", "")
+            
+            # Exact prefix match - it's a tool call
+            if normalized.startswith(normalized_pattern):
+                return 'match'
+            
+            # Could still become this pattern
+            if normalized_pattern.startswith(normalized):
+                return 'partial'
+        
+        # Check if we're still in the opening portion
+        # Allow for whitespace variations like { "name" or {\n"name"
+        if len(normalized) < 8:  # Short buffer, still checking
+            if normalized in ['', '{', '[', '{"', '[{', '{"n', '{"na', '{"nam']:
+                return 'partial'
+        
+        return 'no_match'
+
+
+class JsonInitializationState(BaseState):
+    """
+    Analyzes a potential JSON tool call to determine if it's a known format.
+    
+    This state is entered when a '{' or '[' is detected and JSON parsing is enabled.
+    It buffers characters to check for tool call signatures.
+    """
+    
+    def __init__(self, context: "ParserContext"):
+        super().__init__(context)
+        # Consume the trigger character that caused this transition
+        trigger = self.context.peek_char()
+        self.context.advance()
+        self._signature_buffer = trigger if trigger else ""
+        # Use patterns from config
+        self._checker = JsonToolSignatureChecker(context.json_tool_patterns)
+
+    def run(self) -> None:
+        """
+        Buffer characters and check for tool call signatures.
+        
+        If a match is found, transitions to XmlToolParsingState (JSON mode).
+        If no match, reverts the buffer to text.
+        """
+        from .text_state import TextState
+        from .json_tool_parsing_state import JsonToolParsingState
+        
+        while self.context.has_more_chars():
+            char = self.context.peek_char()
+            self._signature_buffer += char
+            self.context.advance()
+            
+            match = self._checker.check_signature(self._signature_buffer)
+            
+            if match == 'match':
+                # Found a tool signature
+                if self.context.parse_tool_calls:
+                    if self.context.get_current_segment_type() == SegmentType.TEXT:
+                        self.context.emit_segment_end()
+                    # Signature buffer already consumed by this state; pass it along.
+                    self.context.transition_to(
+                        JsonToolParsingState(
+                            self.context,
+                            self._signature_buffer,
+                            signature_consumed=True,
+                        )
+                    )
+                else:
+                    # Tool parsing disabled - emit as text
+                    self.context.append_text_segment(self._signature_buffer)
+                    self.context.transition_to(TextState(self.context))
+                return
+            
+            if match == 'no_match':
+                # Not a tool call - emit as text
+                self.context.append_text_segment(self._signature_buffer)
+                self.context.transition_to(TextState(self.context))
+                return
+            
+            # 'partial' - continue buffering
+
+    def finalize(self) -> None:
+        """
+        Called when stream ends while checking JSON signature.
+        
+        Emit buffered content as text.
+        """
+        from .text_state import TextState
+        
+        if self._signature_buffer:
+            self.context.append_text_segment(self._signature_buffer)
+            self._signature_buffer = ""
+        
+        self.context.transition_to(TextState(self.context))

autobyteus/agent/streaming/parser/states/json_tool_parsing_state.py

@@ -0,0 +1,137 @@
+"""
+JsonToolParsingState: Streams JSON tool call content.
+
+This state identifies JSON tool-call boundaries and streams raw JSON content.
+Tool arguments are parsed later by the ToolInvocationAdapter.
+"""
+from typing import TYPE_CHECKING, List
+
+from .base_state import BaseState
+from ..events import SegmentType
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class JsonToolParsingState(BaseState):
+    """
+    Streams JSON tool call content.
+    
+    Expected formats:
+    - {"name": "tool_name", "arguments": {...}}
+    - [{"name": "tool_name", "arguments": {...}}]
+    
+    Handles nested braces and proper JSON boundary detection.
+    """
+    
+    def __init__(
+        self,
+        context: "ParserContext",
+        signature_buffer: str,
+        signature_consumed: bool = False,
+    ):
+        super().__init__(context)
+        self._signature_buffer = signature_buffer
+        self._signature_consumed = signature_consumed
+        self._brace_count = 0
+        self._bracket_count = 0
+        self._in_string = False
+        self._escape_next = False
+        self._segment_started = False
+        self._initialized = False
+        self._is_array = signature_buffer.startswith('[')
+
+    def run(self) -> None:
+        """
+        Parse JSON tool content, tracking nested braces.
+        """
+        from .text_state import TextState
+        
+        if not self._segment_started:
+            self.context.emit_segment_start(SegmentType.TOOL_CALL)
+            self._segment_started = True
+
+        consumed: List[str] = []
+
+        if not self._initialized:
+            if self._signature_consumed:
+                consumed.append(self._signature_buffer)
+                for char in self._signature_buffer:
+                    self._update_brace_count(char)
+            else:
+                signature = self.context.consume(len(self._signature_buffer))
+                if signature:
+                    consumed.append(signature)
+                    for char in signature:
+                        self._update_brace_count(char)
+            self._initialized = True
+
+        while self.context.has_more_chars():
+            char = self.context.peek_char()
+            self.context.advance()
+            consumed.append(char)
+            self._update_brace_count(char)
+
+            if self._is_json_complete():
+                if consumed:
+                    self.context.emit_segment_content("".join(consumed))
+
+                self.context.emit_segment_end()
+                self.context.transition_to(TextState(self.context))
+                return
+
+        if consumed:
+            self.context.emit_segment_content("".join(consumed))
+
+    def _update_brace_count(self, char: str) -> None:
+        """Update brace/bracket count, handling strings."""
+        if self._escape_next:
+            self._escape_next = False
+            return
+        
+        if char == '\\' and self._in_string:
+            self._escape_next = True
+            return
+        
+        if char == '"' and not self._escape_next:
+            self._in_string = not self._in_string
+            return
+        
+        if self._in_string:
+            return
+        
+        if char == '{':
+            self._brace_count += 1
+        elif char == '}':
+            self._brace_count -= 1
+        elif char == '[':
+            self._bracket_count += 1
+        elif char == ']':
+            self._bracket_count -= 1
+
+    def _is_json_complete(self) -> bool:
+        """Check if we have a complete JSON structure."""
+        if self._in_string:
+            return False
+        
+        if self._is_array:
+            return self._bracket_count == 0 and self._brace_count == 0
+        else:
+            return self._brace_count == 0
+
+    def finalize(self) -> None:
+        """
+        Called when stream ends while parsing JSON.
+        
+        Emit any remaining content and close the segment.
+        """
+        from .text_state import TextState
+        
+        if self.context.has_more_chars():
+            remaining = self.context.consume_remaining()
+            if remaining:
+                self.context.emit_segment_content(remaining)
+
+        if self._segment_started:
+            self.context.emit_segment_end()
+        self.context.transition_to(TextState(self.context))

autobyteus/agent/streaming/parser/states/sentinel_content_state.py

@@ -0,0 +1,30 @@
+"""
+SentinelContentState: Parses sentinel-delimited content blocks.
+"""
+from typing import TYPE_CHECKING, Dict
+
+from .delimited_content_state import DelimitedContentState
+from ..sentinel_format import END_MARKER
+from ..events import SegmentType
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class SentinelContentState(DelimitedContentState):
+    """
+    Parses content for a sentinel block until the matching end marker.
+    """
+
+    def __init__(
+        self,
+        context: "ParserContext",
+        segment_type: SegmentType,
+        metadata: Dict[str, object],
+    ):
+        self.SEGMENT_TYPE = segment_type
+        self._metadata = metadata
+        super().__init__(context, opening_tag="", closing_tag_override=END_MARKER)
+
+    def _get_start_metadata(self) -> dict:
+        return self._metadata

autobyteus/agent/streaming/parser/states/sentinel_initialization_state.py

@@ -0,0 +1,117 @@
+"""
+SentinelInitializationState: Parses sentinel start headers.
+"""
+import json
+from typing import TYPE_CHECKING, Dict, Optional
+
+from .base_state import BaseState
+from ..events import SegmentType
+from ..sentinel_format import START_MARKER, MARKER_END
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class SentinelInitializationState(BaseState):
+    """
+    Parses the sentinel start header and transitions to content state.
+    """
+
+    def __init__(self, context: "ParserContext"):
+        super().__init__(context)
+        self._header_buffer = ""
+
+    def run(self) -> None:
+        from .text_state import TextState
+        from .sentinel_content_state import SentinelContentState
+
+        if not self.context.has_more_chars():
+            return
+
+        start_pos = self.context.get_position()
+        end_idx = self.context.find(MARKER_END, start_pos)
+
+        if end_idx == -1:
+            self._header_buffer += self.context.consume_remaining()
+            if not self._is_possible_prefix(self._header_buffer):
+                self.context.append_text_segment(self._header_buffer)
+                self.context.transition_to(TextState(self.context))
+            return
+
+        self._header_buffer += self.context.consume(end_idx - start_pos + len(MARKER_END))
+
+        if not self._header_buffer.startswith(START_MARKER):
+            self.context.append_text_segment(self._header_buffer)
+            self.context.transition_to(TextState(self.context))
+            return
+
+        header_json = self._header_buffer[len(START_MARKER):]
+        if header_json.endswith(MARKER_END):
+            header_json = header_json[:-len(MARKER_END)]
+        header_json = header_json.strip()
+
+        if not header_json:
+            self.context.append_text_segment(self._header_buffer)
+            self.context.transition_to(TextState(self.context))
+            return
+
+        data = self._parse_header_json(header_json)
+        if not data:
+            self.context.append_text_segment(self._header_buffer)
+            self.context.transition_to(TextState(self.context))
+            return
+
+        type_str = data.get("type")
+        segment_type = self._map_segment_type(type_str)
+
+        if segment_type is None:
+            self.context.append_text_segment(self._header_buffer)
+            self.context.transition_to(TextState(self.context))
+            return
+
+        metadata = dict(data)
+        metadata.pop("type", None)
+
+        if self.context.get_current_segment_type() == SegmentType.TEXT:
+            self.context.emit_segment_end()
+
+        self.context.transition_to(
+            SentinelContentState(
+                self.context,
+                segment_type=segment_type,
+                metadata=metadata,
+            )
+        )
+
+    def finalize(self) -> None:
+        from .text_state import TextState
+
+        if self.context.has_more_chars():
+            self._header_buffer += self.context.consume_remaining()
+
+        if self._header_buffer:
+            self.context.append_text_segment(self._header_buffer)
+        self.context.transition_to(TextState(self.context))
+
+    def _is_possible_prefix(self, buffer: str) -> bool:
+        return START_MARKER.startswith(buffer) or buffer.startswith(START_MARKER)
+
+    def _parse_header_json(self, header_json: str) -> Optional[Dict[str, object]]:
+        try:
+            data = json.loads(header_json)
+            return data if isinstance(data, dict) else None
+        except json.JSONDecodeError:
+            return None
+
+    def _map_segment_type(self, type_str: Optional[str]) -> Optional[SegmentType]:
+        if not type_str:
+            return None
+        value = type_str.strip().lower()
+        mapping = {
+            "text": SegmentType.TEXT,
+            "tool": SegmentType.TOOL_CALL,
+            "tool_call": SegmentType.TOOL_CALL,
+            "write_file": SegmentType.WRITE_FILE,
+            "run_bash": SegmentType.RUN_BASH,
+        }
+        return mapping.get(value)

autobyteus/agent/streaming/parser/states/text_state.py

@@ -0,0 +1,78 @@
+"""
+TextState: Default state for parsing plain text.
+
+This state consumes characters as text content and detects triggers
+for transitioning to specialized parsing states (XML tags, JSON).
+"""
+from typing import TYPE_CHECKING
+
+from .base_state import BaseState
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class TextState(BaseState):
+    """
+    Default state for parsing plain text content.
+    
+    This state:
+    - Accumulates text characters
+    - Detects '<' for potential XML tag transitions
+    - Detects '{' or '[' for potential JSON transitions (if enabled)
+    - Emits text segments when transitioning or at end of buffer
+    """
+    
+    def __init__(self, context: "ParserContext"):
+        super().__init__(context)
+
+    def run(self) -> None:
+        """
+        Process characters as text until a trigger is found or buffer is exhausted.
+        """
+        # Import here to avoid circular dependency
+        from .xml_tag_initialization_state import XmlTagInitializationState
+        
+        start_pos = self.context.get_position()
+
+        if not self.context.has_more_chars():
+            return
+
+        strategies = self.context.detection_strategies
+
+        best_idx = -1
+        best_strategy = None
+        for strategy in strategies:
+            idx = strategy.next_marker(self.context, start_pos)
+            if idx == -1:
+                continue
+            if best_idx == -1 or idx < best_idx:
+                best_idx = idx
+                best_strategy = strategy
+
+        if best_idx == -1:
+            text = self.context.substring(start_pos)
+            if text:
+                self.context.append_text_segment(text)
+            self.context.set_position(self.context.get_buffer_length())
+            return
+
+        if best_idx > start_pos:
+            text = self.context.substring(start_pos, best_idx)
+            if text:
+                self.context.append_text_segment(text)
+
+        self.context.set_position(best_idx)
+
+        if best_strategy is None:
+            self.context.transition_to(XmlTagInitializationState(self.context))
+        else:
+            self.context.transition_to(best_strategy.create_state(self.context))
+        
+    def finalize(self) -> None:
+        """
+        Called when stream ends while in TextState.
+        
+        Nothing special to do here since run() already emits text
+        when buffer is exhausted.
+        """
+        pass