kollabor 0.4.9__py3-none-any.whl → 0.4.15__py3-none-any.whl

core/llm/response_parser.py

@@ -96,7 +96,16 @@ class FileOperationParser:
             re.DOTALL | re.IGNORECASE
         )
 
-        logger.debug("File operation parser initialized with 14 operation patterns")
+        # Special pattern for agent/skill file generation
+        # Uses @@@FILE/@@@END syntax to avoid XML tag conflicts
+        # Format: @@@FILE path/to/file.md\ncontent\n@@@END
+        # Path must contain / to be valid (prevents matching garbage like "@@@FILE blocks.")
+        self.agent_files_pattern = re.compile(
+            r'@@@FILE\s+(\S+/\S+)\n(.*?)@@@END',
+            re.DOTALL
+        )
+
+        logger.debug("File operation parser initialized with 15 operation patterns")
 
     def parse_response(self, llm_response: str) -> List[Dict[str, Any]]:
         """Extract all file operations from LLM response.
@@ -109,7 +118,16 @@ class FileOperationParser:
         """
         operations = []
 
-        # Parse each operation type in order of appearance
+        # FIRST: Parse @@@FILE blocks before any content stripping
+        # This protects inner content from being mangled by tag stripping
+        agent_files_ops, response_without_agent_files = self._parse_agent_files(llm_response)
+        operations.extend(agent_files_ops)
+
+        # Use the cleaned response for remaining parsing
+        llm_response = response_without_agent_files
+
+        # Parse operations that contain <content> blocks from full text first
+        # These need the full response to extract their content properly
         operations.extend(self._parse_operations(
             self.edit_pattern, self._parse_edit_block, llm_response, "edit"
         ))
@@ -121,40 +139,51 @@ class FileOperationParser:
             llm_response, "create_overwrite"
         ))
         operations.extend(self._parse_operations(
-            self.delete_pattern, self._parse_delete_block, llm_response, "delete"
+            self.append_pattern, self._parse_append_block, llm_response, "append"
         ))
         operations.extend(self._parse_operations(
-            self.move_pattern, self._parse_move_block, llm_response, "move"
+            self.insert_after_pattern, self._parse_insert_after_block,
+            llm_response, "insert_after"
         ))
         operations.extend(self._parse_operations(
-            self.copy_pattern, self._parse_copy_block, llm_response, "copy"
+            self.insert_before_pattern, self._parse_insert_before_block,
+            llm_response, "insert_before"
         ))
+
+        # Strip <content>...</content> blocks to avoid parsing XML examples
+        # inside file content as actual commands (e.g., skill files with <read> examples)
+        text_without_content = re.sub(
+            r'<content>.*?</content>',
+            '',
+            llm_response,
+            flags=re.DOTALL | re.IGNORECASE
+        )
+
+        # Parse remaining operations from text with content blocks removed
         operations.extend(self._parse_operations(
-            self.copy_overwrite_pattern, self._parse_copy_overwrite_block,
-            llm_response, "copy_overwrite"
+            self.delete_pattern, self._parse_delete_block, text_without_content, "delete"
         ))
         operations.extend(self._parse_operations(
-            self.append_pattern, self._parse_append_block, llm_response, "append"
+            self.move_pattern, self._parse_move_block, text_without_content, "move"
         ))
         operations.extend(self._parse_operations(
-            self.insert_after_pattern, self._parse_insert_after_block,
-            llm_response, "insert_after"
+            self.copy_pattern, self._parse_copy_block, text_without_content, "copy"
         ))
         operations.extend(self._parse_operations(
-            self.insert_before_pattern, self._parse_insert_before_block,
-            llm_response, "insert_before"
+            self.copy_overwrite_pattern, self._parse_copy_overwrite_block,
+            text_without_content, "copy_overwrite"
         ))
         operations.extend(self._parse_operations(
-            self.mkdir_pattern, self._parse_mkdir_block, llm_response, "mkdir"
+            self.mkdir_pattern, self._parse_mkdir_block, text_without_content, "mkdir"
         ))
         operations.extend(self._parse_operations(
-            self.rmdir_pattern, self._parse_rmdir_block, llm_response, "rmdir"
+            self.rmdir_pattern, self._parse_rmdir_block, text_without_content, "rmdir"
         ))
         operations.extend(self._parse_operations(
-            self.read_pattern, self._parse_read_block, llm_response, "read"
+            self.read_pattern, self._parse_read_block, text_without_content, "read"
         ))
         operations.extend(self._parse_operations(
-            self.grep_pattern, self._parse_grep_block, llm_response, "grep"
+            self.grep_pattern, self._parse_grep_block, text_without_content, "grep"
         ))
 
         if operations:
@@ -162,6 +191,49 @@ class FileOperationParser:
 
         return operations
 
+    def _parse_agent_files(self, llm_response: str) -> Tuple[List[Dict[str, Any]], str]:
+        """Parse @@@FILE blocks and extract file operations.
+
+        Uses simple line-based syntax to avoid XML tag conflicts:
+            @@@FILE path/to/file.md
+            content with <create> <edit> examples - no conflicts
+            @@@END
+
+        This parser runs FIRST to protect inner content from tag stripping.
+
+        Args:
+            llm_response: Raw LLM response text
+
+        Returns:
+            Tuple of (operations list, response with @@@FILE blocks removed)
+        """
+        operations = []
+        cleaned_response = llm_response
+
+        for i, match in enumerate(self.agent_files_pattern.finditer(llm_response)):
+            filepath = match.group(1).strip()
+            content = match.group(2)
+
+            # Remove trailing newline if present (before @@@END)
+            if content.endswith('\n'):
+                content = content[:-1]
+
+            operations.append({
+                "type": "file_create",
+                "id": f"agent_file_{i}",
+                "file": filepath,
+                "content": content
+            })
+            logger.debug(f"Parsed agent file operation: {filepath}")
+
+            # Remove this block from the response
+            cleaned_response = cleaned_response.replace(match.group(0), '', 1)
+
+        if operations:
+            logger.info(f"Parsed {len(operations)} file operations from @@@FILE blocks")
+
+        return operations, cleaned_response
+
     def _parse_operations(
         self,
         pattern: re.Pattern,
@@ -362,6 +434,8 @@ class FileOperationParser:
         """Parse <read> block."""
         file_path = self._extract_tag("file", content).strip()
         lines_spec = self._extract_tag("lines", content, required=False)
+        offset = self._extract_tag("offset", content, required=False)
+        limit = self._extract_tag("limit", content, required=False)
 
         result = {
             "type": "file_read",
@@ -370,6 +444,10 @@ class FileOperationParser:
 
         if lines_spec:
             result["lines"] = lines_spec.strip()
+        if offset:
+            result["offset"] = int(offset.strip())
+        if limit:
+            result["limit"] = int(limit.strip())
 
         return result
 
@@ -422,11 +500,31 @@ class ResponseParser:
             re.DOTALL | re.IGNORECASE
         )
 
+        # Native-style tool_call tags: <tool_call>name</tool_call> or with JSON args
+        # Supports: <tool_call>search_nodes</tool_call>
+        #           <tool_call>{"name": "search_nodes", "arguments": {...}}</tool_call>
+        self.tool_call_pattern = re.compile(
+            r'<tool_call>(.*?)</tool_call>',
+            re.DOTALL | re.IGNORECASE
+        )
+
+        # Question gate tags - suspend tool execution when present
+        self.question_pattern = re.compile(
+            r'<question>(.*?)</question>',
+            re.DOTALL | re.IGNORECASE
+        )
+
         # File operations parser
         self.file_ops_parser = FileOperationParser()
 
         logger.info("Response parser initialized with comprehensive tag support + file operations")
     
+    async def initialize(self) -> bool:
+        """Initialize the response parser."""
+        self.is_initialized = True
+        logger.debug("Response parser async initialization complete")
+        return True
+    
     def parse_response(self, raw_response: str) -> Dict[str, Any]:
         """Parse LLM response and extract all components.
 
@@ -449,13 +547,21 @@ class ResponseParser:
             logger.warning(f"🔍 BUG-011 DIAGNOSTIC: Found {abs(orphaned_closes)} orphaned <think> tags (unclosed)")
 
         # Extract all components
-        thinking_blocks = self._extract_thinking(raw_response)
-        terminal_commands = self._extract_terminal_commands(raw_response)
-        tool_calls = self._extract_tool_calls(raw_response)
+        # IMPORTANT: Parse @@@FILE blocks FIRST to get cleaned response
+        # This prevents XML examples inside @@@FILE from being executed
         file_operations = self.file_ops_parser.parse_response(raw_response)
 
-        # Clean content (remove all tags)
-        clean_content = self._clean_content(raw_response)
+        # Get response with @@@FILE blocks removed for other parsing
+        _, response_without_agent_files = self.file_ops_parser._parse_agent_files(raw_response)
+
+        # Extract other tools from the CLEANED response (not raw)
+        thinking_blocks = self._extract_thinking(response_without_agent_files)
+        terminal_commands = self._extract_terminal_commands(response_without_agent_files)
+        tool_calls = self._extract_tool_calls(response_without_agent_files)
+        question_content = self._extract_question(response_without_agent_files)
+
+        # Clean content (remove all tags) - use cleaned response
+        clean_content = self._clean_content(response_without_agent_files)
 
         # DIAGNOSTIC: Verify defensive fix effectiveness
         if '</think>' in clean_content or '<think>' in clean_content:
@@ -466,29 +572,36 @@ class ResponseParser:
         elif orphaned_closes > 0:
             logger.info(f"✅ BUG-011 SUCCESS: Defensive fix removed {orphaned_closes} orphaned tags")
 
-        # Determine if turn is completed (no tools pending execution)
-        turn_completed = (
-            len(terminal_commands) == 0 and
-            len(tool_calls) == 0 and
-            len(file_operations) == 0
-        )
+        # Count total tools
+        total_tools = len(terminal_commands) + len(tool_calls) + len(file_operations)
+
+        # Question gate: if question present, mark turn as completed but flag tools as pending
+        # This causes the system to stop and wait for user input
+        has_question = question_content is not None
+
+        # Determine if turn is completed
+        # Turn is completed if: no tools OR question present (tools suspended)
+        turn_completed = (total_tools == 0) or has_question
 
         parsed = {
             "raw": raw_response,
             "content": clean_content,
             "turn_completed": turn_completed,
+            "question_gate_active": has_question and total_tools > 0,  # Tools suspended
             "components": {
                 "thinking": thinking_blocks,
                 "terminal_commands": terminal_commands,
                 "tool_calls": tool_calls,
-                "file_operations": file_operations
+                "file_operations": file_operations,
+                "question": question_content
             },
             "metadata": {
                 "has_thinking": bool(thinking_blocks),
                 "has_terminal_commands": bool(terminal_commands),
                 "has_tool_calls": bool(tool_calls),
                 "has_file_operations": bool(file_operations),
-                "total_tools": len(terminal_commands) + len(tool_calls) + len(file_operations),
+                "has_question": has_question,
+                "total_tools": total_tools,
                 "content_length": len(clean_content)
             }
         }
@@ -501,28 +614,53 @@ class ResponseParser:
     
     def _extract_thinking(self, content: str) -> List[str]:
         """Extract thinking content blocks.
-        
+
         Args:
             content: Raw response content
-            
+
         Returns:
             List of thinking content strings
         """
         matches = self.thinking_pattern.findall(content)
         return [match.strip() for match in matches if match.strip()]
+
+    def _extract_question(self, content: str) -> Optional[str]:
+        """Extract question gate content.
+
+        When a <question> tag is present, the agent is asking for user input
+        and all tool calls should be suspended until the user responds.
+
+        Args:
+            content: Raw response content
+
+        Returns:
+            Question content if found, None otherwise
+        """
+        match = self.question_pattern.search(content)
+        if match:
+            return match.group(1).strip()
+        return None
     
     def _extract_terminal_commands(self, content: str) -> List[Dict[str, Any]]:
         """Extract terminal command blocks.
-        
+
         Args:
             content: Raw response content
-            
+
         Returns:
             List of terminal command dictionaries
         """
+        # Strip <content> blocks to avoid parsing terminal examples inside file content
+        text_without_content = re.sub(
+            r'<content>.*?</content>',
+            '',
+            content,
+            flags=re.DOTALL | re.IGNORECASE
+        )
+
         commands = []
-        matches = self.terminal_pattern.findall(content)
-        
+        matches = self.terminal_pattern.findall(text_without_content)
+
         for i, match in enumerate(matches):
             command = match.strip()
             if command:
@@ -532,49 +670,122 @@ class ResponseParser:
                     "command": command,
                     "raw": match
                 })
-        
+
         return commands
     
     def _extract_tool_calls(self, content: str) -> List[Dict[str, Any]]:
-        """Extract MCP tool call blocks.
-        
+        """Extract MCP tool call blocks from both <tool> and <tool_call> tags.
+
+        Supports:
+        - <tool name="tool_name" arg="value">content</tool>
+        - <tool_call>tool_name</tool_call>
+        - <tool_call>{"name": "tool_name", "arguments": {...}}</tool_call>
+
         Args:
             content: Raw response content
-            
+
         Returns:
             List of tool call dictionaries
         """
         tool_calls = []
-        matches = self.tool_pattern.findall(content)
-        
-        for i, (attributes_str, tool_content) in enumerate(matches):
+        tool_index = 0
+
+        # Extract <tool> style calls (attribute-based)
+        for match in self.tool_pattern.finditer(content):
+            attributes_str, tool_content = match.groups()
             try:
-                # Parse tool attributes
                 tool_info = self._parse_tool_attributes(attributes_str)
-                
-                # Build tool call
-                tool_call = {
+                tool_calls.append({
                     "type": "mcp_tool",
-                    "id": f"mcp_tool_{i}",
+                    "id": f"mcp_tool_{tool_index}",
                     "name": tool_info.get("name", "unknown"),
                     "arguments": tool_info.get("arguments", {}),
                     "content": tool_content.strip(),
-                    "raw": f"<tool {attributes_str}>{tool_content}</tool>"
-                }
-                
+                    "raw": match.group(0)
+                })
+                tool_index += 1
+            except Exception as e:
+                logger.warning(f"Failed to parse <tool> call: {e}")
+                tool_calls.append({
+                    "type": "malformed_tool",
+                    "id": f"malformed_{tool_index}",
+                    "error": str(e),
+                    "raw": match.group(0)
+                })
+                tool_index += 1
+
+        # Extract <tool_call> style calls (content-based)
+        for match in self.tool_call_pattern.finditer(content):
+            call_content = match.group(1).strip()
+            try:
+                tool_call = self._parse_tool_call_content(call_content, tool_index)
                 tool_calls.append(tool_call)
-                
+                tool_index += 1
             except Exception as e:
-                logger.warning(f"Failed to parse tool call: {e}")
-                # Add as malformed tool call for debugging
+                logger.warning(f"Failed to parse <tool_call> content: {e}")
                 tool_calls.append({
                     "type": "malformed_tool",
-                    "id": f"malformed_{i}",
+                    "id": f"malformed_{tool_index}",
                     "error": str(e),
-                    "raw": f"<tool {attributes_str}>{tool_content}</tool>"
+                    "raw": match.group(0)
                 })
-        
+                tool_index += 1
+
         return tool_calls
+
+    def _parse_tool_call_content(self, content: str, index: int) -> Dict[str, Any]:
+        """Parse content from <tool_call> tags.
+
+        Supports:
+        - Simple name: "search_nodes"
+        - JSON format: {"name": "search_nodes", "arguments": {"query": "test"}}
+
+        Args:
+            content: Content between <tool_call> tags
+            index: Tool index for ID generation
+
+        Returns:
+            Parsed tool call dictionary
+        """
+        content = content.strip()
+
+        # Try JSON format first
+        if content.startswith("{"):
+            try:
+                data = json.loads(content)
+                return {
+                    "type": "mcp_tool",
+                    "id": f"mcp_tool_{index}",
+                    "name": data.get("name", "unknown"),
+                    "arguments": data.get("arguments", {}),
+                    "content": "",
+                    "raw": f"<tool_call>{content}</tool_call>"
+                }
+            except json.JSONDecodeError:
+                pass
+
+        # Simple name format: just the tool name, maybe with inline args
+        # Handle: "search_nodes" or "search_nodes query=test"
+        parts = content.split(None, 1)
+        tool_name = parts[0] if parts else "unknown"
+        arguments = {}
+
+        # Parse inline arguments if present
+        if len(parts) > 1:
+            arg_str = parts[1]
+            # Try to parse key=value pairs
+            for pair in re.findall(r'(\w+)=(["\']?)([^"\'=\s]+)\2', arg_str):
+                key, _, value = pair
+                arguments[key] = self._convert_value(value)
+
+        return {
+            "type": "mcp_tool",
+            "id": f"mcp_tool_{index}",
+            "name": tool_name,
+            "arguments": arguments,
+            "content": "",
+            "raw": f"<tool_call>{content}</tool_call>"
+        }
     
     def _parse_tool_attributes(self, attributes_str: str) -> Dict[str, Any]:
         """Parse tool tag attributes.
@@ -661,6 +872,13 @@ class ResponseParser:
         # Remove tool tags but preserve content structure
         cleaned = self.tool_pattern.sub('', cleaned)
 
+        # Remove <tool_call> tags
+        cleaned = self.tool_call_pattern.sub('', cleaned)
+
+        # Remove question tags but preserve content for display
+        # The question content stays visible, just the tags are removed
+        cleaned = self.question_pattern.sub(r'\1', cleaned)
+
         # Remove file operation tags (all 14 types)
         # Only successfully parsed tags are removed; malformed tags remain visible
         cleaned = self.file_ops_parser.edit_pattern.sub('', cleaned)

core/llm/response_processor.py

@@ -33,6 +33,12 @@ class ResponseProcessor:
         
         logger.info("Response processor initialized")
     
+    async def initialize(self) -> bool:
+        """Initialize the response processor."""
+        self.is_initialized = True
+        logger.debug("Response processor async initialization complete")
+        return True
+    
     def process_response(self, raw_response: str) -> Dict[str, Any]:
         """Process raw LLM response.
         
@@ -89,14 +95,21 @@ class ResponseProcessor:
     
     def _extract_terminal_commands(self, content: str) -> List[str]:
         """Extract terminal commands from content.
-        
+
         Args:
             content: Raw content with potential terminal tags
-            
+
         Returns:
             List of terminal commands
         """
-        matches = self.terminal_pattern.findall(content)
+        # Strip <content> blocks to avoid parsing terminal examples inside file content
+        text_without_content = re.sub(
+            r'<content>.*?</content>',
+            '',
+            content,
+            flags=re.DOTALL | re.IGNORECASE
+        )
+        matches = self.terminal_pattern.findall(text_without_content)
         return [match.strip() for match in matches if match.strip()]
     
     def _extract_tool_calls(self, content: str) -> List[Dict[str, Any]]:

core/llm/tool_executor.py

@@ -264,7 +264,7 @@ class ToolExecutor:
             
             try:
                 stdout, stderr = await asyncio.wait_for(
-                    process.communicate(), 
+                    process.communicate(),
                     timeout=self.terminal_timeout
                 )
             except asyncio.TimeoutError:
@@ -276,6 +276,11 @@ class ToolExecutor:
                     success=False,
                     error=f"Command timed out after {self.terminal_timeout} seconds"
                 )
+            except asyncio.CancelledError:
+                # Clean up subprocess on cancellation to avoid ResourceWarning
+                process.kill()
+                await process.wait()
+                raise  # Re-raise to propagate cancellation
             
             # Process results
             stdout_text = stdout.decode('utf-8', errors='replace')

core/logging/__init__.py

@@ -5,6 +5,7 @@ from .setup import (
     setup_from_config,
     get_current_config,
     is_configured,
+    set_level,
     CompactFormatter,
     LoggingSetup
 )
@@ -14,6 +15,7 @@ __all__ = [
     'setup_from_config',
     'get_current_config',
     'is_configured',
+    'set_level',
     'CompactFormatter',
     'LoggingSetup'
 ]

core/logging/setup.py

@@ -11,6 +11,8 @@ import threading
 from pathlib import Path
 from typing import Any, Dict, Optional
 
+from ..utils.config_utils import get_logs_dir
+
 
 class CompactFormatter(logging.Formatter):
     """Custom formatter that compacts level names and includes file location."""
@@ -37,12 +39,10 @@ class LoggingSetup:
         """Setup minimal logging before config system is available.
 
         Args:
-            log_dir: Optional log directory, defaults to .kollabor-cli/logs
+            log_dir: Optional log directory, defaults to project-specific logs
         """
         if log_dir is None:
-            from ..utils.config_utils import get_config_directory
-            config_dir = get_config_directory()
-            log_dir = config_dir / "logs"
+            log_dir = get_logs_dir()
 
         # Ensure log directory exists
         log_dir.mkdir(parents=True, exist_ok=True)
@@ -96,7 +96,8 @@ class LoggingSetup:
 
         # Extract configuration values
         level = logging_config.get('level', 'INFO').upper()
-        log_file = logging_config.get('file', '.kollabor-cli/logs/kollabor.log')
+        default_log_path = get_logs_dir() / "kollabor.log"
+        log_file = logging_config.get('file') or str(default_log_path)
         format_type = logging_config.get('format_type', 'compact')
         custom_format = logging_config.get('format', None)
 
@@ -183,6 +184,28 @@ class LoggingSetup:
         """Check if logging has been configured."""
         return self._configured
 
+    def set_level(self, level: str) -> None:
+        """Set logging level at runtime (hot reload support).
+
+        Args:
+            level: Log level string (DEBUG, INFO, WARNING, ERROR)
+        """
+        level = level.upper()
+        numeric_level = getattr(logging, level, logging.INFO)
+
+        # Update root logger
+        logging.getLogger().setLevel(numeric_level)
+
+        # Update all existing loggers
+        for logger_name in logging.Logger.manager.loggerDict:
+            existing_logger = logging.getLogger(logger_name)
+            existing_logger.setLevel(numeric_level)
+
+        # Update current config
+        self._current_config['level'] = level
+
+        logging.info(f"Logging level changed to {level}")
+
 
 # Global instance
 logging_setup = LoggingSetup()
@@ -205,4 +228,9 @@ def get_current_config() -> Dict[str, Any]:
 
 def is_configured() -> bool:
     """Check if logging has been configured."""
-    return logging_setup.is_configured()
+    return logging_setup.is_configured()
+
+
+def set_level(level: str) -> None:
+    """Set logging level at runtime (hot reload support)."""
+    logging_setup.set_level(level)