tunacode-cli 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl

tunacode/tools/grep.py

@@ -12,13 +12,17 @@ CLAUDE_ANCHOR[grep-module]: Fast parallel file search with 3-second deadline
 """
 
 import asyncio
+import logging
 import re
-import subprocess
 import time
 from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
+import defusedxml.ElementTree as ET
+
+from tunacode.configuration.defaults import DEFAULT_USER_CONFIG
 from tunacode.exceptions import TooBroadPatternError, ToolExecutionError
 from tunacode.tools.base import BaseTool
 from tunacode.tools.grep_components import (
@@ -28,6 +32,10 @@ from tunacode.tools.grep_components import (
     SearchResult,
 )
 from tunacode.tools.grep_components.result_formatter import ResultFormatter
+from tunacode.utils.ripgrep import RipgrepExecutor
+from tunacode.utils.ripgrep import metrics as ripgrep_metrics
+
+logger = logging.getLogger(__name__)
 
 
 class ParallelGrep(BaseTool):
@@ -42,11 +50,144 @@ class ParallelGrep(BaseTool):
         self._file_filter = FileFilter()
         self._pattern_matcher = PatternMatcher()
         self._result_formatter = ResultFormatter()
+        self._ripgrep_executor = RipgrepExecutor()
+
+        # Load configuration
+        self._config = self._load_ripgrep_config()
 
     @property
     def tool_name(self) -> str:
         return "grep"
 
+    @lru_cache(maxsize=1)
+    def _get_base_prompt(self) -> str:
+        """Load and return the base prompt from XML file.
+
+        Returns:
+            str: The loaded prompt from XML or a default prompt
+        """
+        try:
+            # Load prompt from XML file
+            prompt_file = Path(__file__).parent / "prompts" / "grep_prompt.xml"
+            if prompt_file.exists():
+                tree = ET.parse(prompt_file)
+                root = tree.getroot()
+                description = root.find("description")
+                if description is not None:
+                    return description.text.strip()
+        except Exception as e:
+            logger.warning(f"Failed to load XML prompt for grep: {e}")
+
+        # Fallback to default prompt
+        return """A powerful search tool built on ripgrep
+
+Usage:
+- ALWAYS use Grep for search tasks. NEVER invoke `grep` or `rg` as a Bash command.
+- Supports full regex syntax
+- Filter files with glob or type parameters
+- Multiple output modes available"""
+
+    @lru_cache(maxsize=1)
+    def _get_parameters_schema(self) -> Dict[str, Any]:
+        """Get the parameters schema for grep tool.
+
+        Returns:
+            Dict containing the JSON schema for tool parameters
+        """
+        # Try to load from XML first
+        try:
+            prompt_file = Path(__file__).parent / "prompts" / "grep_prompt.xml"
+            if prompt_file.exists():
+                tree = ET.parse(prompt_file)
+                root = tree.getroot()
+                parameters = root.find("parameters")
+                if parameters is not None:
+                    schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []}
+                    required_fields: List[str] = []
+
+                    for param in parameters.findall("parameter"):
+                        name = param.get("name")
+                        required = param.get("required", "false").lower() == "true"
+                        param_type = param.find("type")
+                        description = param.find("description")
+
+                        if name and param_type is not None:
+                            prop = {
+                                "type": param_type.text.strip(),
+                                "description": description.text.strip()
+                                if description is not None
+                                else "",
+                            }
+
+                            # Add enum values if present
+                            enums = param.findall("enum")
+                            if enums:
+                                prop["enum"] = [e.text.strip() for e in enums]
+
+                            schema["properties"][name] = prop
+                            if required:
+                                required_fields.append(name)
+
+                    schema["required"] = required_fields
+                    return schema
+        except Exception as e:
+            logger.warning(f"Failed to load parameters from XML for grep: {e}")
+
+        # Fallback to hardcoded schema
+        return {
+            "type": "object",
+            "properties": {
+                "pattern": {
+                    "type": "string",
+                    "description": "Regular expression pattern to search for",
+                },
+                "directory": {"type": "string", "description": "Directory to search in"},
+                "include": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "File patterns to include",
+                },
+                "exclude": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "File patterns to exclude",
+                },
+                "max_results": {"type": "integer", "description": "Maximum number of results"},
+                "context_before": {
+                    "type": "integer",
+                    "description": "Lines of context before matches",
+                },
+                "context_after": {
+                    "type": "integer",
+                    "description": "Lines of context after matches",
+                },
+            },
+            "required": ["pattern"],
+        }
+
+    def _load_ripgrep_config(self) -> Dict:
+        """Load ripgrep configuration from settings."""
+        try:
+            settings = DEFAULT_USER_CONFIG.get("settings", {})
+            return settings.get(
+                "ripgrep",
+                {
+                    "timeout": 10,
+                    "max_buffer_size": 1048576,
+                    "max_results": 100,
+                    "enable_metrics": False,
+                    "debug": False,
+                },
+            )
+        except Exception:
+            return {
+                "timeout": 10,
+                "max_buffer_size": 1048576,
+                "max_results": 100,
+                "enable_metrics": False,
+                "debug": False,
+            }
+
     async def _execute(
         self,
         pattern: str,
@@ -176,98 +317,106 @@ class ParallelGrep(BaseTool):
         self, pattern: str, candidates: List[Path], config: SearchConfig
     ) -> List[SearchResult]:
         """
-        Run ripgrep on pre-filtered file list with first match deadline.
+        Run ripgrep on pre-filtered file list using the enhanced RipgrepExecutor.
         """
 
-        def run_ripgrep_filtered():
-            cmd = ["rg", "--json"]
+        def run_enhanced_ripgrep():
+            """Execute ripgrep search using the new executor."""
+            start_time = time.time()
+            first_match_time = None
+            results = []
 
-            # Add configuration flags
-            if not config.case_sensitive:
-                cmd.append("--ignore-case")
-            if config.context_lines > 0:
-                cmd.extend(["--context", str(config.context_lines)])
-            if config.max_results:
-                cmd.extend(["--max-count", str(config.max_results)])
+            # Configure timeout from settings
+            timeout = min(self._config.get("timeout", 10), config.timeout_seconds)
 
-            # Add pattern and explicit file list
-            cmd.append(pattern)
-            cmd.extend(str(f) for f in candidates)
+            # If ripgrep executor is using fallback, skip this method entirely
+            if self._ripgrep_executor._use_python_fallback:
+                # Return empty to trigger Python fallback in the calling function
+                return []
 
             try:
-                # Start the process
-                process = subprocess.Popen(
-                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+                # Use the enhanced executor with support for context lines
+                # Note: Currently searching all files, not using candidates
+                # This is a limitation that should be addressed in future enhancement
+                search_results = self._ripgrep_executor.search(
+                    pattern=pattern,
+                    path=".",  # Search in current directory
+                    timeout=timeout,
+                    max_matches=config.max_results,
+                    case_insensitive=not config.case_sensitive,
+                    context_before=config.context_lines,
+                    context_after=config.context_lines,
                 )
 
-                # Monitor for first match within deadline
-                start_time = time.time()
-                output_lines = []
-                first_match_found = False
+                # Track first match time for metrics
+                if search_results and first_match_time is None:
+                    first_match_time = time.time() - start_time
 
-                while True:
                     # Check if we exceeded the first match deadline
-                    if (
-                        not first_match_found
-                        and (time.time() - start_time) > config.first_match_deadline
-                    ):
-                        process.kill()
-                        process.wait()
+                    if first_match_time > config.first_match_deadline:
+                        if self._config.get("debug", False):
+                            logger.debug(
+                                f"Search exceeded first match deadline: {first_match_time:.2f}s"
+                            )
                         raise TooBroadPatternError(pattern, config.first_match_deadline)
 
-                    # Check if process is still running
-                    if process.poll() is not None:
-                        # Process finished, get any remaining output
-                        remaining_output, _ = process.communicate()
-                        if remaining_output:
-                            output_lines.extend(remaining_output.splitlines())
-                        break
-
-                    # Try to read a line (non-blocking)
-                    try:
-                        # Use a small timeout to avoid blocking indefinitely
-                        line = process.stdout.readline()
-                        if line:
-                            output_lines.append(line.rstrip())
-                            # Check if this is a match line
-                            if '"type":"match"' in line:
-                                first_match_found = True
-                    except Exception:
-                        pass
-
-                    # Small sleep to avoid busy waiting
-                    time.sleep(0.01)
-
-                # Check exit code
-                if process.returncode == 0 or output_lines:
-                    # Return output even if exit code is non-zero but we have matches
-                    return "\n".join(output_lines)
-                else:
-                    return None
+                # Parse results
+                for result_line in search_results:
+                    # Parse ripgrep output format "file:line:content"
+                    parts = result_line.split(":", 2)
+                    if len(parts) >= 3:
+                        # Filter to only include results from candidates
+                        file_path = Path(parts[0])
+                        if file_path not in candidates:
+                            continue
+
+                        try:
+                            search_result = SearchResult(
+                                file_path=parts[0],
+                                line_number=int(parts[1]),
+                                line_content=parts[2] if len(parts) > 2 else "",
+                                match_start=0,
+                                match_end=len(parts[2]) if len(parts) > 2 else 0,
+                                context_before=[],
+                                context_after=[],
+                                relevance_score=1.0,
+                            )
+                            results.append(search_result)
+
+                            # Stop if we have enough results
+                            if config.max_results and len(results) >= config.max_results:
+                                break
+                        except (ValueError, IndexError):
+                            continue
 
             except TooBroadPatternError:
                 raise
-            except (subprocess.TimeoutExpired, FileNotFoundError):
-                return None
-            except Exception:
-                # Make sure to clean up the process
-                if "process" in locals():
-                    try:
-                        process.kill()
-                        process.wait()
-                    except Exception:
-                        pass
-                return None
-
-        # Run ripgrep with monitoring in thread pool
+            except Exception as e:
+                if self._config.get("debug", False):
+                    logger.debug(f"Search error: {e}")
+                # Return empty to trigger fallback
+                return []
+
+            # Record metrics if enabled
+            if self._config.get("enable_metrics", False):
+                total_time = time.time() - start_time
+                ripgrep_metrics.record_search(
+                    duration=total_time, used_fallback=self._ripgrep_executor._use_python_fallback
+                )
+
+                if self._config.get("debug", False):
+                    logger.debug(
+                        f"Ripgrep search completed in {total_time:.2f}s "
+                        f"(first match: {first_match_time:.2f}s if found)"
+                    )
+
+            return results
+
+        # Run the enhanced ripgrep search
         try:
-            output = await asyncio.get_event_loop().run_in_executor(
-                self._executor, run_ripgrep_filtered
+            return await asyncio.get_event_loop().run_in_executor(
+                self._executor, run_enhanced_ripgrep
             )
-            if output:
-                parsed = self._pattern_matcher.parse_ripgrep_output(output)
-                return parsed
-            return []
         except TooBroadPatternError:
             raise
 

tunacode/tools/grep_components/result_formatter.py

@@ -1,21 +1,53 @@
 """
+Extended result formatter with multiple output modes for flexible presentation.
 Result formatting functionality for the grep tool.
 """
 
-from typing import List
+from typing import Dict, List
 
 from .search_result import SearchConfig, SearchResult
 
 
 class ResultFormatter:
-    """Handles formatting of search results for display."""
+    """Handles formatting of search results for display with multiple output modes."""
 
     @staticmethod
-    def format_results(results: List[SearchResult], pattern: str, config: SearchConfig) -> str:
-        """Format search results for display."""
+    def format_results(
+        results: List[SearchResult],
+        pattern: str,
+        config: SearchConfig,
+        output_mode: str = "content",
+    ) -> str:
+        """Format search results for display.
+
+        Args:
+            results: List of search results
+            pattern: Search pattern
+            config: Search configuration
+            output_mode: Output format mode:
+                - "content": Show matching lines with context (default)
+                - "files_with_matches": Show only file paths
+                - "count": Show match counts per file
+                - "json": JSON format for programmatic use
+
+        Returns:
+            Formatted string based on output mode
+        """
         if not results:
             return f"No matches found for pattern: {pattern}"
 
+        if output_mode == "files_with_matches":
+            return ResultFormatter._format_files_only(results, pattern)
+        elif output_mode == "count":
+            return ResultFormatter._format_count(results, pattern)
+        elif output_mode == "json":
+            return ResultFormatter._format_json(results, pattern)
+        else:  # Default to "content"
+            return ResultFormatter._format_content(results, pattern, config)
+
+    @staticmethod
+    def _format_content(results: List[SearchResult], pattern: str, config: SearchConfig) -> str:
+        """Format results with full content and context."""
         output = []
         output.append(f"Found {len(results)} matches for pattern: {pattern}")
         output.append("=" * 60)
@@ -43,3 +75,65 @@ class ResultFormatter:
                 output.append(f"  {line_num:4d}│ {context_line}")
 
         return "\n".join(output)
+
+    @staticmethod
+    def _format_files_only(results: List[SearchResult], pattern: str) -> str:
+        """Format results showing only file paths."""
+        # Collect unique file paths
+        files = sorted(set(r.file_path for r in results))
+
+        output = []
+        output.append(f"Files with matches for pattern: {pattern}")
+        output.append(f"Total files: {len(files)}")
+        output.append("=" * 60)
+
+        for file_path in files:
+            output.append(file_path)
+
+        return "\n".join(output)
+
+    @staticmethod
+    def _format_count(results: List[SearchResult], pattern: str) -> str:
+        """Format results showing match counts per file."""
+        # Count matches per file
+        file_counts: Dict[str, int] = {}
+        for result in results:
+            file_counts[result.file_path] = file_counts.get(result.file_path, 0) + 1
+
+        output = []
+        output.append(f"Match counts for pattern: {pattern}")
+        output.append(f"Total matches: {len(results)} across {len(file_counts)} files")
+        output.append("=" * 60)
+
+        # Sort by count (descending) then by file path
+        sorted_counts = sorted(file_counts.items(), key=lambda x: (-x[1], x[0]))
+
+        for file_path, count in sorted_counts:
+            output.append(f"{count:5d} {file_path}")
+
+        return "\n".join(output)
+
+    @staticmethod
+    def _format_json(results: List[SearchResult], pattern: str) -> str:
+        """Format results as JSON for programmatic use."""
+        import json
+
+        # Convert results to JSON-serializable format
+        json_results = []
+        for result in results:
+            json_results.append(
+                {
+                    "file": result.file_path,
+                    "line": result.line_number,
+                    "content": result.line_content,
+                    "match_start": result.match_start,
+                    "match_end": result.match_end,
+                    "context_before": result.context_before,
+                    "context_after": result.context_after,
+                    "score": result.relevance_score,
+                }
+            )
+
+        output_data = {"pattern": pattern, "total_matches": len(results), "results": json_results}
+
+        return json.dumps(output_data, indent=2)

tunacode/tools/list_dir.py

@@ -6,14 +6,20 @@ Provides efficient directory listing without using shell commands.
 """
 
 import asyncio
+import logging
 import os
+from functools import lru_cache
 from pathlib import Path
-from typing import List, Tuple
+from typing import Any, Dict, List, Tuple
+
+import defusedxml.ElementTree as ET
 
 from tunacode.exceptions import ToolExecutionError
 from tunacode.tools.base import FileBasedTool
 from tunacode.types import FilePath, ToolResult
 
+logger = logging.getLogger(__name__)
+
 
 class ListDirTool(FileBasedTool):
     """Tool for listing directory contents without shell commands."""
@@ -22,6 +28,92 @@ class ListDirTool(FileBasedTool):
     def tool_name(self) -> str:
         return "ListDir"
 
+    @lru_cache(maxsize=1)
+    def _get_base_prompt(self) -> str:
+        """Load and return the base prompt from XML file.
+
+        Returns:
+            str: The loaded prompt from XML or a default prompt
+        """
+        try:
+            # Load prompt from XML file
+            prompt_file = Path(__file__).parent / "prompts" / "list_dir_prompt.xml"
+            if prompt_file.exists():
+                tree = ET.parse(prompt_file)
+                root = tree.getroot()
+                description = root.find("description")
+                if description is not None:
+                    return description.text.strip()
+        except Exception as e:
+            logger.warning(f"Failed to load XML prompt for list_dir: {e}")
+
+        # Fallback to default prompt
+        return """Lists files and directories in a given path"""
+
+    @lru_cache(maxsize=1)
+    def _get_parameters_schema(self) -> Dict[str, Any]:
+        """Get the parameters schema for list_dir tool.
+
+        Returns:
+            Dict containing the JSON schema for tool parameters
+        """
+        # Try to load from XML first
+        try:
+            prompt_file = Path(__file__).parent / "prompts" / "list_dir_prompt.xml"
+            if prompt_file.exists():
+                tree = ET.parse(prompt_file)
+                root = tree.getroot()
+                parameters = root.find("parameters")
+                if parameters is not None:
+                    schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []}
+                    required_fields: List[str] = []
+
+                    for param in parameters.findall("parameter"):
+                        name = param.get("name")
+                        required = param.get("required", "false").lower() == "true"
+                        param_type = param.find("type")
+                        description = param.find("description")
+
+                        if name and param_type is not None:
+                            prop = {
+                                "type": param_type.text.strip(),
+                                "description": description.text.strip()
+                                if description is not None
+                                else "",
+                            }
+
+                            # Handle array types
+                            if param_type.text.strip() == "array":
+                                items = param.find("items")
+                                if items is not None:
+                                    prop["items"] = {"type": items.text.strip()}
+
+                            schema["properties"][name] = prop
+                            if required:
+                                required_fields.append(name)
+
+                    schema["required"] = required_fields
+                    return schema
+        except Exception as e:
+            logger.warning(f"Failed to load parameters from XML for list_dir: {e}")
+
+        # Fallback to hardcoded schema
+        return {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "description": "The absolute path to the directory to list",
+                },
+                "ignore": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "List of glob patterns to ignore",
+                },
+            },
+            "required": ["path"],
+        }
+
     async def _execute(
         self, directory: FilePath = ".", max_entries: int = 200, show_hidden: bool = False
     ) -> ToolResult:
@@ -48,7 +140,34 @@ class ListDirTool(FileBasedTool):
         if not dir_path.is_dir():
             raise NotADirectoryError(f"Not a directory: {dir_path}")
 
-        # Collect entries in a background thread to prevent blocking the event loop
+        # Try to use cached data from CodeIndex first
+        try:
+            from tunacode.core.code_index import CodeIndex
+
+            index = CodeIndex.get_instance()
+            cached_entries = index.get_directory_contents(dir_path)
+
+            if cached_entries:
+                # Filter cached entries based on show_hidden
+                if not show_hidden:
+                    cached_entries = [name for name in cached_entries if not name.startswith(".")]
+
+                # Limit entries and format output
+                limited_entries = cached_entries[:max_entries]
+
+                # Return simple format for cached results (names only for speed)
+                if limited_entries:
+                    return f"Files in {dir_path}:\n" + "\n".join(
+                        f"  {name}" for name in limited_entries
+                    )
+                else:
+                    return f"Directory {dir_path} is empty"
+
+        except Exception as e:
+            # If CodeIndex fails, fall back to regular scanning
+            logger.debug(f"CodeIndex cache miss for {dir_path}: {e}")
+
+        # Fallback: Collect entries in a background thread to prevent blocking the event loop
         def _scan_directory(path: Path) -> List[Tuple[str, bool, str]]:
             """Synchronous helper that scans a directory and returns entry metadata."""
             collected: List[Tuple[str, bool, str]] = []
@@ -98,6 +217,17 @@ class ListDirTool(FileBasedTool):
         # Sort entries: directories first, then files, both alphabetically
         entries.sort(key=lambda x: (not x[1], x[0].lower()))
 
+        # Update CodeIndex cache with the fresh data
+        try:
+            from tunacode.core.code_index import CodeIndex
+
+            index = CodeIndex.get_instance()
+            # Extract just the names for cache storage
+            entry_names = [name for name, _, _ in entries]
+            index.update_directory_cache(dir_path, entry_names)
+        except Exception as e:
+            logger.debug(f"Failed to update CodeIndex cache for {dir_path}: {e}")
+
         # Apply limit after sorting to ensure consistent results
         total_entries = len(entries)
         if len(entries) > max_entries: