tunacode-cli 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

tunacode/tools/grep.py

@@ -6,6 +6,7 @@ This tool provides sophisticated grep-like functionality with:
 - Multiple search strategies (literal, regex, fuzzy)
 - Smart result ranking and deduplication
 - Context-aware output formatting
+- Timeout handling for overly broad patterns (3 second deadline for first match)
 """
 
 import asyncio
@@ -13,12 +14,13 @@ import fnmatch
 import os
 import re
 import subprocess
+import time
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Union
 
-from tunacode.exceptions import ToolExecutionError
+from tunacode.exceptions import TooBroadPatternError, ToolExecutionError
 from tunacode.tools.base import BaseTool
 
 
@@ -48,6 +50,7 @@ class SearchConfig:
     exclude_patterns: List[str] = None
     max_file_size: int = 1024 * 1024  # 1MB
     timeout_seconds: int = 30
+    first_match_deadline: float = 3.0  # Timeout for finding first match
 
 
 # Fast-Glob Prefilter Configuration
@@ -146,6 +149,7 @@ class ParallelGrep(BaseTool):
         max_results: int = 50,
         context_lines: int = 2,
         search_type: str = "smart",  # smart, ripgrep, python, hybrid
+        return_format: str = "string",  # "string" (default) or "list" (legacy)
     ) -> str:
         """
         Execute parallel grep search with fast-glob prefiltering and multiple strategies.
@@ -174,6 +178,8 @@ class ParallelGrep(BaseTool):
             )
 
             if not candidates:
+                if return_format == "list":
+                    return []
                 return f"No files found matching pattern: {include_pattern}"
 
             # 2️⃣ Smart strategy selection based on candidate count
@@ -203,6 +209,7 @@ class ParallelGrep(BaseTool):
             )
 
             # 4️⃣ Execute chosen strategy with pre-filtered candidates
+            # Execute search with pre-filtered candidates
             if search_type == "ripgrep":
                 results = await self._ripgrep_search_filtered(pattern, candidates, config)
             elif search_type == "python":
@@ -216,151 +223,37 @@ class ParallelGrep(BaseTool):
             strategy_info = f"Strategy: {search_type} (was {original_search_type}), Files: {len(candidates)}/{MAX_GLOB}"
             formatted_results = self._format_results(results, pattern, config)
 
-            # Add strategy info to results
-            if formatted_results.startswith("Found"):
-                lines = formatted_results.split("\n")
-                lines[1] = (
-                    f"Strategy: {search_type} | Candidates: {len(candidates)} files | " + lines[1]
-                )
-                return "\n".join(lines)
+            if return_format == "list":
+                # Legacy: return list of file paths with at least one match
+                file_set = set()
+                for r in results:
+                    file_set.add(r.file_path)
+                return list(file_set)
             else:
-                return f"{formatted_results}\n\n{strategy_info}"
+                # Add strategy info to results
+                if formatted_results.startswith("Found"):
+                    lines = formatted_results.split("\n")
+                    lines[1] = (
+                        f"Strategy: {search_type} | Candidates: {len(candidates)} files | "
+                        + lines[1]
+                    )
+                    return "\n".join(lines)
+                else:
+                    return f"{formatted_results}\n\n{strategy_info}"
 
+        except TooBroadPatternError:
+            # Re-raise TooBroadPatternError without wrapping it
+            raise
         except Exception as e:
             raise ToolExecutionError(f"Grep search failed: {str(e)}")
 
-    async def _smart_search(
-        self, pattern: str, directory: str, config: SearchConfig
-    ) -> List[SearchResult]:
-        """Smart search that chooses optimal strategy based on context."""
-
-        # Try ripgrep first (fastest for large codebases)
-        try:
-            results = await self._ripgrep_search(pattern, directory, config)
-            if results:
-                return results
-        except Exception:
-            pass
-
-        # Fallback to Python implementation
-        return await self._python_search(pattern, directory, config)
-
-    async def _ripgrep_search(
-        self, pattern: str, directory: str, config: SearchConfig
-    ) -> List[SearchResult]:
-        """Use ripgrep for high-performance searching."""
-
-        def run_ripgrep():
-            cmd = ["rg", "--json"]
-
-            # Add options based on config
-            if not config.case_sensitive:
-                cmd.append("--ignore-case")
-            if config.context_lines > 0:
-                cmd.extend(["--context", str(config.context_lines)])
-            if config.max_results:
-                cmd.extend(["--max-count", str(config.max_results)])
-
-            # Add include/exclude patterns
-            for pattern_str in config.include_patterns:
-                if pattern_str != "*":
-                    cmd.extend(["--glob", pattern_str])
-            for pattern_str in config.exclude_patterns:
-                cmd.extend(["--glob", f"!{pattern_str}"])
-
-            # Add pattern and directory
-            cmd.extend([pattern, directory])
-
-            try:
-                result = subprocess.run(
-                    cmd, capture_output=True, text=True, timeout=config.timeout_seconds
-                )
-                return result.stdout if result.returncode == 0 else None
-            except (subprocess.TimeoutExpired, FileNotFoundError):
-                return None
-
-        # Run ripgrep in thread pool
-        output = await asyncio.get_event_loop().run_in_executor(self._executor, run_ripgrep)
-
-        if not output:
-            return []
-
-        # Parse ripgrep JSON output
-        return self._parse_ripgrep_output(output)
-
-    async def _python_search(
-        self, pattern: str, directory: str, config: SearchConfig
-    ) -> List[SearchResult]:
-        """Pure Python parallel search implementation."""
-
-        # Find all files to search
-        files = await self._find_files(directory, config)
-
-        # Prepare search pattern
-        if config.use_regex:
-            flags = 0 if config.case_sensitive else re.IGNORECASE
-            regex_pattern = re.compile(pattern, flags)
-        else:
-            regex_pattern = None
-
-        # Create search tasks for parallel execution
-        search_tasks = []
-        for file_path in files:
-            task = self._search_file(file_path, pattern, regex_pattern, config)
-            search_tasks.append(task)
-
-        # Execute searches in parallel
-        all_results = await asyncio.gather(*search_tasks, return_exceptions=True)
-
-        # Flatten results and filter out exceptions
-        results = []
-        for file_results in all_results:
-            if isinstance(file_results, list):
-                results.extend(file_results)
-
-        # Sort by relevance and limit results
-        results.sort(key=lambda r: r.relevance_score, reverse=True)
-        return results[: config.max_results]
-
-    async def _hybrid_search(
-        self, pattern: str, directory: str, config: SearchConfig
-    ) -> List[SearchResult]:
-        """Hybrid approach using multiple search methods concurrently."""
-
-        # Run multiple search strategies in parallel
-        tasks = [
-            self._ripgrep_search(pattern, directory, config),
-            self._python_search(pattern, directory, config),
-        ]
-
-        results_list = await asyncio.gather(*tasks, return_exceptions=True)
-
-        # Merge and deduplicate results
-        all_results = []
-        for results in results_list:
-            if isinstance(results, list):
-                all_results.extend(results)
-
-        # Deduplicate by file path and line number
-        seen = set()
-        unique_results = []
-        for result in all_results:
-            key = (result.file_path, result.line_number)
-            if key not in seen:
-                seen.add(key)
-                unique_results.append(result)
-
-        # Sort and limit
-        unique_results.sort(key=lambda r: r.relevance_score, reverse=True)
-        return unique_results[: config.max_results]
-
-    # ====== NEW FILTERED SEARCH METHODS ======
+    # ====== SEARCH METHODS ======
 
     async def _ripgrep_search_filtered(
         self, pattern: str, candidates: List[Path], config: SearchConfig
     ) -> List[SearchResult]:
         """
-        Run ripgrep on pre-filtered file list.
+        Run ripgrep on pre-filtered file list with first match deadline.
         """
 
         def run_ripgrep_filtered():
@@ -379,25 +272,87 @@ class ParallelGrep(BaseTool):
             cmd.extend(str(f) for f in candidates)
 
             try:
-                result = subprocess.run(
-                    cmd, capture_output=True, text=True, timeout=config.timeout_seconds
+                # Start the process
+                process = subprocess.Popen(
+                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
                 )
-                return result.stdout if result.returncode == 0 else None
+
+                # Monitor for first match within deadline
+                start_time = time.time()
+                output_lines = []
+                first_match_found = False
+
+                while True:
+                    # Check if we exceeded the first match deadline
+                    if (
+                        not first_match_found
+                        and (time.time() - start_time) > config.first_match_deadline
+                    ):
+                        process.kill()
+                        process.wait()
+                        raise TooBroadPatternError(pattern, config.first_match_deadline)
+
+                    # Check if process is still running
+                    if process.poll() is not None:
+                        # Process finished, get any remaining output
+                        remaining_output, _ = process.communicate()
+                        if remaining_output:
+                            output_lines.extend(remaining_output.splitlines())
+                        break
+
+                    # Try to read a line (non-blocking)
+                    try:
+                        # Use a small timeout to avoid blocking indefinitely
+                        line = process.stdout.readline()
+                        if line:
+                            output_lines.append(line.rstrip())
+                            # Check if this is a match line
+                            if '"type":"match"' in line:
+                                first_match_found = True
+                    except Exception:
+                        pass
+
+                    # Small sleep to avoid busy waiting
+                    time.sleep(0.01)
+
+                # Check exit code
+                if process.returncode == 0 or output_lines:
+                    # Return output even if exit code is non-zero but we have matches
+                    return "\n".join(output_lines)
+                else:
+                    return None
+
+            except TooBroadPatternError:
+                raise
             except (subprocess.TimeoutExpired, FileNotFoundError):
                 return None
+            except Exception:
+                # Make sure to clean up the process
+                if "process" in locals():
+                    try:
+                        process.kill()
+                        process.wait()
+                    except Exception:
+                        pass
+                return None
 
-        # Run ripgrep in thread pool
-        output = await asyncio.get_event_loop().run_in_executor(
-            self._executor, run_ripgrep_filtered
-        )
-
-        return self._parse_ripgrep_output(output) if output else []
+        # Run ripgrep with monitoring in thread pool
+        try:
+            output = await asyncio.get_event_loop().run_in_executor(
+                self._executor, run_ripgrep_filtered
+            )
+            if output:
+                parsed = self._parse_ripgrep_output(output)
+                return parsed
+            return []
+        except TooBroadPatternError:
+            raise
 
     async def _python_search_filtered(
         self, pattern: str, candidates: List[Path], config: SearchConfig
     ) -> List[SearchResult]:
         """
-        Run Python parallel search on pre-filtered candidates.
+        Run Python parallel search on pre-filtered candidates with first match deadline.
         """
         # Prepare search pattern
         if config.use_regex:
@@ -406,24 +361,63 @@ class ParallelGrep(BaseTool):
         else:
             regex_pattern = None
 
+        # Track search progress
+        first_match_event = asyncio.Event()
+
+        async def search_with_monitoring(file_path: Path):
+            """Search a file and signal when first match is found."""
+            try:
+                file_results = await self._search_file(file_path, pattern, regex_pattern, config)
+                if file_results and not first_match_event.is_set():
+                    first_match_event.set()
+                return file_results
+            except Exception:
+                return []
+
         # Create search tasks for candidates only
         search_tasks = []
         for file_path in candidates:
-            task = self._search_file(file_path, pattern, regex_pattern, config)
+            task = search_with_monitoring(file_path)
             search_tasks.append(task)
 
-        # Execute searches in parallel
-        all_results = await asyncio.gather(*search_tasks, return_exceptions=True)
+        # Create a deadline task
+        async def check_deadline():
+            """Monitor for first match deadline."""
+            await asyncio.sleep(config.first_match_deadline)
+            if not first_match_event.is_set():
+                # Cancel all pending tasks
+                for task in search_tasks:
+                    if not task.done():
+                        task.cancel()
+                raise TooBroadPatternError(pattern, config.first_match_deadline)
 
-        # Flatten results and filter out exceptions
-        results = []
-        for file_results in all_results:
-            if isinstance(file_results, list):
-                results.extend(file_results)
+        deadline_task = asyncio.create_task(check_deadline())
+
+        try:
+            # Execute searches in parallel with deadline monitoring
+            all_results = await asyncio.gather(*search_tasks, return_exceptions=True)
 
-        # Sort by relevance and limit results
-        results.sort(key=lambda r: r.relevance_score, reverse=True)
-        return results[: config.max_results]
+            # Cancel deadline task if we got results
+            deadline_task.cancel()
+
+            # Flatten results and filter out exceptions
+            results = []
+            for file_results in all_results:
+                if isinstance(file_results, list):
+                    results.extend(file_results)
+
+            # Sort by relevance and limit results
+            results.sort(key=lambda r: r.relevance_score, reverse=True)
+            return results[: config.max_results]
+
+        except asyncio.CancelledError:
+            # Re-raise TooBroadPatternError if that's what caused the cancellation
+            if deadline_task.done():
+                try:
+                    await deadline_task
+                except TooBroadPatternError:
+                    raise
+            return []
 
     async def _hybrid_search_filtered(
         self, pattern: str, candidates: List[Path], config: SearchConfig
@@ -440,6 +434,14 @@ class ParallelGrep(BaseTool):
 
         results_list = await asyncio.gather(*tasks, return_exceptions=True)
 
+        # Check if any task raised TooBroadPatternError
+        too_broad_errors = [r for r in results_list if isinstance(r, TooBroadPatternError)]
+        if too_broad_errors:
+            # If both strategies timed out, raise the error
+            valid_results = [r for r in results_list if isinstance(r, list)]
+            if not valid_results:
+                raise too_broad_errors[0]
+
         # Merge and deduplicate results
         all_results = []
         for results in results_list:
@@ -459,42 +461,6 @@ class ParallelGrep(BaseTool):
         unique_results.sort(key=lambda r: r.relevance_score, reverse=True)
         return unique_results[: config.max_results]
 
-    async def _find_files(self, directory: str, config: SearchConfig) -> List[Path]:
-        """Find all files matching include/exclude patterns."""
-
-        def find_files_sync():
-            files = []
-            dir_path = Path(directory)
-
-            for file_path in dir_path.rglob("*"):
-                if not file_path.is_file():
-                    continue
-
-                # Check file size
-                try:
-                    if file_path.stat().st_size > config.max_file_size:
-                        continue
-                except OSError:
-                    continue
-
-                # Check include patterns
-                if not any(
-                    fnmatch.fnmatch(str(file_path), pattern) for pattern in config.include_patterns
-                ):
-                    continue
-
-                # Check exclude patterns
-                if any(
-                    fnmatch.fnmatch(str(file_path), pattern) for pattern in config.exclude_patterns
-                ):
-                    continue
-
-                files.append(file_path)
-
-            return files
-
-        return await asyncio.get_event_loop().run_in_executor(self._executor, find_files_sync)
-
     async def _search_file(
         self,
         file_path: Path,
@@ -676,6 +642,7 @@ class ParallelGrep(BaseTool):
 async def grep(
     pattern: str,
     directory: str = ".",
+    path: Optional[str] = None,  # Alias for directory
     case_sensitive: bool = False,
     use_regex: bool = False,
     include_files: Optional[str] = None,
@@ -683,7 +650,8 @@ async def grep(
     max_results: int = 50,
     context_lines: int = 2,
     search_type: str = "smart",
-) -> str:
+    return_format: str = "string",
+) -> Union[str, List[str]]:
     """
     Advanced parallel grep search with multiple strategies.
 
@@ -706,6 +674,10 @@ async def grep(
         grep("function.*export", "src/", use_regex=True, include_files="*.js,*.ts")
         grep("import.*pandas", ".", include_files="*.py", search_type="hybrid")
     """
+    # Handle path alias for directory
+    if path is not None:
+        directory = path
+
     tool = ParallelGrep()
     return await tool._execute(
         pattern=pattern,
@@ -717,4 +689,5 @@ async def grep(
         max_results=max_results,
         context_lines=context_lines,
         search_type=search_type,
+        return_format=return_format,
     )

tunacode/tools/list_dir.py

@@ -0,0 +1,190 @@
+"""
+Module: tunacode.tools.list_dir
+
+Directory listing tool for agent operations in the TunaCode application.
+Provides efficient directory listing without using shell commands.
+"""
+
+import asyncio
+import os
+from pathlib import Path
+from typing import List, Tuple
+
+from tunacode.exceptions import ToolExecutionError
+from tunacode.tools.base import FileBasedTool
+from tunacode.types import FilePath, ToolResult
+
+
+class ListDirTool(FileBasedTool):
+    """Tool for listing directory contents without shell commands."""
+
+    @property
+    def tool_name(self) -> str:
+        return "ListDir"
+
+    async def _execute(
+        self, directory: FilePath = ".", max_entries: int = 200, show_hidden: bool = False
+    ) -> ToolResult:
+        """List the contents of a directory.
+
+        Args:
+            directory: The path to the directory to list (defaults to current directory)
+            max_entries: Maximum number of entries to return (default: 200)
+            show_hidden: Whether to include hidden files/directories (default: False)
+
+        Returns:
+            ToolResult: Formatted list of files and directories
+
+        Raises:
+            Exception: Directory access errors
+        """
+        # Convert to Path object for easier handling
+        dir_path = Path(directory).resolve()
+
+        # Verify it's a directory
+        if not dir_path.exists():
+            raise FileNotFoundError(f"Directory not found: {dir_path}")
+
+        if not dir_path.is_dir():
+            raise NotADirectoryError(f"Not a directory: {dir_path}")
+
+        # Collect entries in a background thread to prevent blocking the event loop
+        def _scan_directory(path: Path) -> List[Tuple[str, bool, str]]:
+            """Synchronous helper that scans a directory and returns entry metadata."""
+            collected: List[Tuple[str, bool, str]] = []
+            try:
+                with os.scandir(path) as scanner:
+                    for entry in scanner:
+                        # Skip hidden files if requested
+                        if not show_hidden and entry.name.startswith("."):
+                            continue
+
+                        try:
+                            is_directory = entry.is_dir(follow_symlinks=False)
+                            is_symlink = entry.is_symlink()
+
+                            # Determine type indicator
+                            if is_symlink:
+                                type_indicator = "@"  # Symlink
+                            elif is_directory:
+                                type_indicator = "/"  # Directory
+                            elif entry.is_file():
+                                # Check if executable
+                                if os.access(entry.path, os.X_OK):
+                                    type_indicator = "*"  # Executable
+                                else:
+                                    type_indicator = ""  # Regular file
+                            else:
+                                type_indicator = "?"  # Unknown type
+
+                            collected.append((entry.name, is_directory, type_indicator))
+
+                        except (OSError, PermissionError):
+                            # If we can't stat the entry, include it with unknown type
+                            collected.append((entry.name, False, "?"))
+            except PermissionError:
+                # Re-raise for the outer async context to handle uniformly
+                raise
+
+            return collected
+
+        try:
+            entries: List[Tuple[str, bool, str]] = await asyncio.to_thread(
+                _scan_directory, dir_path
+            )
+        except PermissionError as e:
+            raise PermissionError(f"Permission denied accessing directory: {dir_path}") from e
+
+        # Sort entries: directories first, then files, both alphabetically
+        entries.sort(key=lambda x: (not x[1], x[0].lower()))
+
+        # Apply limit after sorting to ensure consistent results
+        total_entries = len(entries)
+        if len(entries) > max_entries:
+            entries = entries[:max_entries]
+
+        # Format output
+        if not entries:
+            return f"Directory '{dir_path}' is empty"
+
+        # Build formatted output
+        lines = [f"Contents of '{dir_path}':"]
+        lines.append("")
+
+        # Determine column width for better formatting
+        max_name_length = max(len(name) for name, _, _ in entries)
+        col_width = min(max_name_length + 2, 50)  # Cap at 50 chars
+
+        for name, is_dir, type_indicator in entries:
+            # Truncate long names
+            display_name = name
+            if len(name) > 47:
+                display_name = name[:44] + "..."
+
+            # Add type indicator
+            display_name += type_indicator
+
+            # Add entry type description
+            if is_dir:
+                entry_type = "[DIR]"
+            else:
+                entry_type = "[FILE]"
+
+            lines.append(f"  {display_name:<{col_width}} {entry_type}")
+
+        # Add summary
+        displayed_count = len(entries)
+        dir_count = sum(1 for _, is_dir, _ in entries if is_dir)
+        file_count = displayed_count - dir_count
+
+        lines.append("")
+        lines.append(
+            f"Total: {displayed_count} entries ({dir_count} directories, {file_count} files)"
+        )
+
+        if total_entries > max_entries:
+            lines.append(f"Note: Output limited to {max_entries} entries")
+
+        return "\n".join(lines)
+
+    def _format_args(self, directory: FilePath = ".", *args, **kwargs) -> str:
+        """Format arguments for display."""
+        all_args = [repr(str(directory))]
+
+        # Add other keyword arguments if present
+        for key, value in kwargs.items():
+            if key not in ["max_entries", "show_hidden"]:
+                continue
+            all_args.append(f"{key}={repr(value)}")
+
+        return ", ".join(all_args)
+
+    def _get_error_context(self, directory: FilePath = None, *args, **kwargs) -> str:
+        """Get error context including directory path."""
+        if directory:
+            return f"listing directory '{directory}'"
+        return super()._get_error_context(*args, **kwargs)
+
+
+# Create the function that maintains compatibility with pydantic-ai
+async def list_dir(directory: str = ".", max_entries: int = 200, show_hidden: bool = False) -> str:
+    """
+    List the contents of a directory without using shell commands.
+
+    Uses os.scandir for efficient directory listing with proper error handling.
+    Results are sorted with directories first, then files, both alphabetically.
+
+    Args:
+        directory: The path to the directory to list (defaults to current directory)
+        max_entries: Maximum number of entries to return (default: 200)
+        show_hidden: Whether to include hidden files/directories (default: False)
+
+    Returns:
+        str: Formatted list of directory contents or error message
+    """
+    tool = ListDirTool(None)  # No UI for pydantic-ai compatibility
+    try:
+        return await tool.execute(directory, max_entries=max_entries, show_hidden=show_hidden)
+    except ToolExecutionError as e:
+        # Return error message for pydantic-ai compatibility
+        return str(e)