autobyteus 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

autobyteus/agent/streaming/parser/parser_context.py

@@ -0,0 +1,227 @@
+"""
+ParserContext: Holds the shared state for the streaming parser state machine.
+
+This class manages the scanner, current state, and configuration.
+Event emission is delegated to the EventEmitter.
+States use this context to read characters, emit events, and transition.
+"""
+from typing import Optional, List, Dict, Any, TYPE_CHECKING
+
+from .stream_scanner import StreamScanner
+from .event_emitter import EventEmitter
+from .events import SegmentEvent, SegmentType
+
+if TYPE_CHECKING:
+    from .states.base_state import BaseState
+    from .json_parsing_strategies.base import JsonToolParsingStrategy
+
+
+class ParserConfig:
+    """Configuration for the streaming parser."""
+    
+    # Default patterns for JSON tool call detection
+    DEFAULT_JSON_PATTERNS = [
+        '{"tool"',
+        '{"tool_calls"',
+        '{"tools"',
+        '{"function"',
+        '{"name"',
+        '[{"tool"',
+        '[{"function"',
+        '[{"name"'
+    ]
+    
+    def __init__(
+        self,
+        parse_tool_calls: bool = True,
+        json_tool_patterns: Optional[List[str]] = None,
+        json_tool_parser: Optional["JsonToolParsingStrategy"] = None,
+        strategy_order: Optional[List[str]] = None,
+        segment_id_prefix: Optional[str] = None,
+    ):
+        self.parse_tool_calls = parse_tool_calls
+        self.json_tool_patterns = json_tool_patterns or self.DEFAULT_JSON_PATTERNS.copy()
+        self.json_tool_parser = json_tool_parser
+        self.strategy_order = strategy_order or ["xml_tag"]
+        self.segment_id_prefix = segment_id_prefix
+
+
+class ParserContext:
+    """
+    Holds the shared state for the streaming parser state machine.
+    
+    This context provides:
+    - Scanner for reading the character stream
+    - EventEmitter for segment events
+    - State management for transitions
+    - Configuration access
+    """
+    
+    def __init__(self, config: Optional[ParserConfig] = None):
+        """
+        Initialize the parser context.
+        
+        Args:
+            config: Parser configuration. Uses defaults if not provided.
+        """
+        self._config = config or ParserConfig()
+        self._scanner = StreamScanner()
+        self._emitter = EventEmitter(segment_id_prefix=self._config.segment_id_prefix)
+        self._current_state: Optional["BaseState"] = None
+        from .strategies.registry import create_detection_strategies
+        self._strategies = create_detection_strategies(self._config.strategy_order)
+
+    @property
+    def config(self) -> ParserConfig:
+        """Get the parser configuration."""
+        return self._config
+
+    @property
+    def parse_tool_calls(self) -> bool:
+        """Whether to parse tool calls."""
+        return self._config.parse_tool_calls
+
+    @property
+    def json_tool_patterns(self) -> List[str]:
+        """Get the JSON tool call patterns."""
+        return self._config.json_tool_patterns
+
+    @property
+    def json_tool_parser(self) -> Optional["JsonToolParsingStrategy"]:
+        """Get the JSON tool parser strategy if configured."""
+        return self._config.json_tool_parser
+
+    @property
+    def detection_strategies(self):
+        """Get the ordered detection strategies."""
+        return self._strategies
+
+    # --- State Management ---
+    
+    @property
+    def current_state(self) -> "BaseState":
+        """Get the current state."""
+        if self._current_state is None:
+            raise RuntimeError("No current state is set.")
+        return self._current_state
+
+    @current_state.setter
+    def current_state(self, state: "BaseState") -> None:
+        """Set the current state."""
+        self._current_state = state
+
+    def transition_to(self, new_state: "BaseState") -> None:
+        """Transition to a new state."""
+        self._current_state = new_state
+
+    # --- Scanner Delegation ---
+    
+    def append(self, text: str) -> None:
+        """Append text to the scanner buffer."""
+        self._scanner.append(text)
+
+    def peek_char(self) -> Optional[str]:
+        """Peek at the current character without advancing."""
+        return self._scanner.peek()
+
+    def advance(self) -> None:
+        """Advance the cursor by one position."""
+        self._scanner.advance()
+
+    def advance_by(self, count: int) -> None:
+        """Advance the cursor by multiple positions."""
+        self._scanner.advance_by(count)
+
+    def has_more_chars(self) -> bool:
+        """Check if there are more characters to read."""
+        return self._scanner.has_more_chars()
+
+    def get_position(self) -> int:
+        """Get the current cursor position."""
+        return self._scanner.get_position()
+
+    def get_buffer_length(self) -> int:
+        """Get the total buffer length."""
+        return self._scanner.get_buffer_length()
+
+    def set_position(self, position: int) -> None:
+        """Set the cursor position."""
+        self._scanner.set_position(position)
+
+    def rewind_by(self, count: int) -> None:
+        """
+        Rewind the cursor by a specified number of positions.
+        
+        This is an explicit helper for the common rewind-and-transition pattern.
+        
+        Args:
+            count: Number of positions to rewind.
+        """
+        new_pos = max(0, self._scanner.get_position() - count)
+        self._scanner.set_position(new_pos)
+
+    def substring(self, start: int, end: Optional[int] = None) -> str:
+        """Extract a substring from the buffer."""
+        return self._scanner.substring(start, end)
+
+    def find(self, sub: str, start: Optional[int] = None) -> int:
+        """Find a substring in the buffer."""
+        return self._scanner.find(sub, start)
+
+    def consume(self, count: int) -> str:
+        """Consume a number of characters from the buffer."""
+        return self._scanner.consume(count)
+
+    def consume_remaining(self) -> str:
+        """Consume all remaining characters from the buffer."""
+        return self._scanner.consume_remaining()
+
+    def compact(self, min_prefix: int = 65536) -> None:
+        """Compact the scanner buffer."""
+        self._scanner.compact(min_prefix=min_prefix)
+
+    # --- Event Emission (Delegated to EventEmitter) ---
+    
+    def emit_segment_start(self, segment_type: SegmentType, **metadata) -> str:
+        """Emit a SEGMENT_START event."""
+        return self._emitter.emit_segment_start(segment_type, **metadata)
+
+    def emit_segment_content(self, delta: Any) -> None:
+        """Emit a SEGMENT_CONTENT event."""
+        self._emitter.emit_segment_content(delta)
+
+    def emit_segment_end(self) -> Optional[str]:
+        """Emit a SEGMENT_END event."""
+        return self._emitter.emit_segment_end()
+
+    def get_current_segment_id(self) -> Optional[str]:
+        """Get the ID of the currently active segment."""
+        return self._emitter.get_current_segment_id()
+
+    def get_current_segment_type(self) -> Optional[SegmentType]:
+        """Get the type of the currently active segment."""
+        return self._emitter.get_current_segment_type()
+
+    def get_current_segment_content(self) -> str:
+        """Get the accumulated content of the current segment."""
+        return self._emitter.get_current_segment_content()
+
+    def get_current_segment_metadata(self) -> Dict[str, Any]:
+        """Get the metadata of the current segment."""
+        return self._emitter.get_current_segment_metadata()
+
+    def update_current_segment_metadata(self, **metadata) -> None:
+        """Update metadata for the current segment."""
+        self._emitter.update_current_segment_metadata(**metadata)
+
+    def get_and_clear_events(self) -> List[SegmentEvent]:
+        """Get all queued events and clear the queue."""
+        return self._emitter.get_and_clear_events()
+
+    def get_events(self) -> List[SegmentEvent]:
+        """Get all queued events without clearing."""
+        return self._emitter.get_events()
+
+    def append_text_segment(self, text: str) -> None:
+        """Append text content to the current text segment, starting one if needed."""
+        self._emitter.append_text_segment(text)

autobyteus/agent/streaming/parser/parser_factory.py

@@ -0,0 +1,132 @@
+"""
+Parser factory and strategy selection for streaming parsers.
+
+Selects a parser implementation based on configuration or environment.
+"""
+from __future__ import annotations
+
+import os
+from typing import Callable, Dict, List, Optional, Protocol, TYPE_CHECKING
+
+from .events import SegmentEvent
+from .parser_context import ParserConfig
+from .streaming_parser import StreamingParser
+
+
+class StreamingParserProtocol(Protocol):
+    """Protocol for streaming parsers used by the response handler."""
+
+    @property
+    def config(self) -> ParserConfig:
+        ...
+
+    def feed(self, chunk: str) -> List[SegmentEvent]:
+        ...
+
+    def finalize(self) -> List[SegmentEvent]:
+        ...
+
+
+ENV_PARSER_NAME = "AUTOBYTEUS_STREAM_PARSER"
+DEFAULT_PARSER_NAME = "xml"
+
+if TYPE_CHECKING:
+    from .json_parsing_strategies.base import JsonToolParsingStrategy
+
+
+def _clone_config(
+    config: Optional[ParserConfig],
+    *,
+    parse_tool_calls: Optional[bool] = None,
+    json_tool_patterns: Optional[List[str]] = None,
+    json_tool_parser: Optional["JsonToolParsingStrategy"] = None,
+    strategy_order: Optional[List[str]] = None,
+    segment_id_prefix: Optional[str] = None,
+) -> ParserConfig:
+    base = config or ParserConfig()
+    return ParserConfig(
+        parse_tool_calls=base.parse_tool_calls if parse_tool_calls is None else parse_tool_calls,
+        json_tool_patterns=(
+            base.json_tool_patterns.copy()
+            if json_tool_patterns is None
+            else json_tool_patterns
+        ),
+        json_tool_parser=(
+            base.json_tool_parser
+            if json_tool_parser is None
+            else json_tool_parser
+        ),
+        strategy_order=(
+            base.strategy_order.copy()
+            if strategy_order is None
+            else strategy_order
+        ),
+        segment_id_prefix=(
+            base.segment_id_prefix
+            if segment_id_prefix is None
+            else segment_id_prefix
+        ),
+    )
+
+
+def _build_xml(config: Optional[ParserConfig]) -> StreamingParserProtocol:
+    xml_config = _clone_config(
+        config,
+        parse_tool_calls=True,
+        strategy_order=["xml_tag"],
+    )
+    return StreamingParser(config=xml_config)
+
+
+def _build_json(config: Optional[ParserConfig]) -> StreamingParserProtocol:
+    json_config = _clone_config(
+        config,
+        parse_tool_calls=True,
+        strategy_order=["json_tool"],
+    )
+    return StreamingParser(config=json_config)
+
+
+def _build_api_tool_call(config: Optional[ParserConfig]) -> StreamingParserProtocol:
+    # API tool calls handled elsewhere; keep tag parsing but disable tool parsing.
+    api_tool_call_config = _clone_config(config, parse_tool_calls=False)
+    return StreamingParser(config=api_tool_call_config)
+
+
+def _build_sentinel(config: Optional[ParserConfig]) -> StreamingParserProtocol:
+    sentinel_config = _clone_config(
+        config,
+        strategy_order=["sentinel"],
+        parse_tool_calls=True,
+    )
+    return StreamingParser(config=sentinel_config)
+
+
+PARSER_REGISTRY: Dict[str, Callable[[Optional[ParserConfig]], StreamingParserProtocol]] = {
+    "xml": _build_xml,
+    "json": _build_json,
+    "api_tool_call": _build_api_tool_call,
+    "sentinel": _build_sentinel,
+}
+
+
+def resolve_parser_name(explicit_name: Optional[str] = None) -> str:
+    """Resolve parser name from explicit value or environment."""
+    name = explicit_name or os.getenv(ENV_PARSER_NAME, DEFAULT_PARSER_NAME)
+    return name.strip().lower()
+
+
+def create_streaming_parser(
+    config: Optional[ParserConfig] = None,
+    *,
+    parser_name: Optional[str] = None,
+) -> StreamingParserProtocol:
+    """Create a streaming parser based on configuration or environment."""
+    name = resolve_parser_name(parser_name)
+    builder = PARSER_REGISTRY.get(name)
+    if builder is None:
+        raise ValueError(
+            f"Unknown parser strategy '{name}'. "
+            f"Supported: {', '.join(sorted(PARSER_REGISTRY))}."
+        )
+    return builder(config)

autobyteus/agent/streaming/parser/sentinel_format.py

@@ -0,0 +1,7 @@
+"""
+Sentinel format constants for streaming parser.
+"""
+
+START_MARKER = "[[SEG_START"
+END_MARKER = "[[SEG_END]]"
+MARKER_END = "]]"

autobyteus/agent/streaming/parser/state_factory.py

@@ -0,0 +1,62 @@
+"""
+StateFactory: Factory for creating parser states.
+
+This pattern helps avoid circular imports between states by providing
+a central location for state instantiation.
+"""
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .parser_context import ParserContext
+    from .states.base_state import BaseState
+
+
+class StateFactory:
+    """
+    Factory for creating parser state instances.
+    
+    This avoids circular imports between state classes by providing
+    lazy imports at the point of creation.
+    """
+    
+    @staticmethod
+    def text_state(context: "ParserContext") -> "BaseState":
+        """Create a TextState instance."""
+        from .states.text_state import TextState
+        return TextState(context)
+    
+    @staticmethod
+    def xml_tag_init_state(context: "ParserContext") -> "BaseState":
+        """Create an XmlTagInitializationState instance."""
+        from .states.xml_tag_initialization_state import XmlTagInitializationState
+        return XmlTagInitializationState(context)
+    
+    @staticmethod
+    def write_file_parsing_state(context: "ParserContext", opening_tag: str) -> "BaseState":
+        """Create a CustomXmlTagWriteFileParsingState instance."""
+        from .states.custom_xml_tag_write_file_parsing_state import CustomXmlTagWriteFileParsingState
+        return CustomXmlTagWriteFileParsingState(context, opening_tag)
+    
+    @staticmethod
+    def run_bash_parsing_state(context: "ParserContext", opening_tag: str) -> "BaseState":
+        """Create a CustomXmlTagRunBashParsingState instance."""
+        from .states.custom_xml_tag_run_bash_parsing_state import CustomXmlTagRunBashParsingState
+        return CustomXmlTagRunBashParsingState(context, opening_tag)
+    
+    @staticmethod
+    def xml_tool_parsing_state(context: "ParserContext", signature_buffer: str) -> "BaseState":
+        """Create a XmlToolParsingState instance."""
+        from .states.xml_tool_parsing_state import XmlToolParsingState
+        return XmlToolParsingState(context, signature_buffer)
+    
+    @staticmethod
+    def json_init_state(context: "ParserContext") -> "BaseState":
+        """Create a JsonInitializationState instance."""
+        from .states.json_initialization_state import JsonInitializationState
+        return JsonInitializationState(context)
+    
+    @staticmethod
+    def json_tool_parsing_state(context: "ParserContext", signature_buffer: str) -> "BaseState":
+        """Create a JsonToolParsingState instance."""
+        from .states.json_tool_parsing_state import JsonToolParsingState
+        return JsonToolParsingState(context, signature_buffer)

autobyteus/agent/streaming/parser/states/__init__.py

@@ -0,0 +1 @@
+# States subpackage

autobyteus/agent/streaming/parser/states/base_state.py

@@ -0,0 +1,60 @@
+"""
+BaseState: Abstract base class for parser state machine states.
+
+All states in the streaming parser inherit from this class and implement
+the run() and finalize() methods.
+"""
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class BaseState(ABC):
+    """
+    Abstract base class for all parser states.
+    
+    Each state is responsible for:
+    1. Processing characters from the context's scanner
+    2. Emitting appropriate SegmentEvents
+    3. Transitioning to other states when triggers are detected
+    """
+    
+    def __init__(self, context: "ParserContext"):
+        """
+        Initialize the state with a reference to the parser context.
+        
+        Args:
+            context: The shared parser context.
+        """
+        self._context = context
+
+    @property
+    def context(self) -> "ParserContext":
+        """Get the parser context."""
+        return self._context
+
+    @abstractmethod
+    def run(self) -> None:
+        """
+        Main processing loop for this state.
+        
+        This method should:
+        - Read characters from context.peek_char()
+        - Emit events via context.emit_segment_*()
+        - Transition to other states via context.transition_to()
+        - Return when no more characters or when transitioning
+        """
+        pass
+
+    def finalize(self) -> None:
+        """
+        Called when the stream ends while in this state.
+        
+        This method should clean up any pending buffers and ensure
+        all accumulated content is properly emitted as events.
+        
+        Default implementation does nothing. Override if needed.
+        """
+        pass

autobyteus/agent/streaming/parser/states/custom_xml_tag_run_bash_parsing_state.py

@@ -0,0 +1,38 @@
+"""
+RunBashParsingState: Parses <run_bash>...</run_bash> blocks.
+
+Simplified implementation that parses terminal commands.
+"""
+from typing import TYPE_CHECKING
+
+from .delimited_content_state import DelimitedContentState
+from ..events import SegmentType
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class CustomXmlTagRunBashParsingState(DelimitedContentState):
+    """
+    Parses terminal command blocks.
+    
+    Supported format: <run_bash>command</run_bash>
+    
+    The state:
+    1. Emits SEGMENT_START (no metadata)
+    2. Streams command content as SEGMENT_CONTENT events
+    3. Emits SEGMENT_END when </run_bash> is found
+    """
+    
+    CLOSING_TAG = "</run_bash>"
+    SEGMENT_TYPE = SegmentType.RUN_BASH
+    
+    def __init__(self, context: "ParserContext", opening_tag: str):
+        """
+        Initialize the run_bash parsing state.
+        
+        Args:
+            context: The parser context.
+            opening_tag: The opening tag (e.g., '<run_bash>').
+        """
+        super().__init__(context, opening_tag)

autobyteus/agent/streaming/parser/states/custom_xml_tag_write_file_parsing_state.py

@@ -0,0 +1,55 @@
+"""
+WriteFileParsingState: Parses <write_file path="...">...</write_file> blocks.
+
+This state handles the extraction of file path from the tag attributes
+and streams the file content until the closing </write_file> tag is found.
+"""
+import re
+from typing import TYPE_CHECKING, Optional
+
+from .delimited_content_state import DelimitedContentState
+from ..events import SegmentType
+
+if TYPE_CHECKING:
+    from ..parser_context import ParserContext
+
+
+class CustomXmlTagWriteFileParsingState(DelimitedContentState):
+    """
+    Parses write_file content blocks.
+    
+    Expected format: <write_file path="...">content</write_file>
+    
+    The state:
+    1. Extracts the path attribute from the opening tag
+    2. Emits SEGMENT_START with path metadata
+    3. Streams content characters as SEGMENT_CONTENT events
+    4. Emits SEGMENT_END when </write_file> is found
+    """
+    
+    # Pattern to extract path from <write_file path="..."> or <write_file path='...'>
+    PATH_PATTERN = re.compile(r'path\s*=\s*["\']([^"\']+)["\']', re.IGNORECASE)
+    CLOSING_TAG = "</write_file>"
+    SEGMENT_TYPE = SegmentType.WRITE_FILE
+    
+    def __init__(self, context: "ParserContext", opening_tag: str):
+        """
+        Initialize the write_file parsing state.
+        
+        Args:
+            context: The parser context.
+            opening_tag: The complete opening tag (e.g., '<write_file path="/a.py">').
+        """
+        super().__init__(context, opening_tag)
+        self._file_path: Optional[str] = None
+        
+        # Extract file path from opening tag
+        match = self.PATH_PATTERN.search(opening_tag)
+        if match:
+            self._file_path = match.group(1)
+
+    def _can_start_segment(self) -> bool:
+        return self._file_path is not None
+
+    def _get_start_metadata(self) -> dict:
+        return {"path": self._file_path} if self._file_path else {}