code-puppy 0.0.214__py3-none-any.whl → 0.0.366__py3-none-any.whl

code_puppy/tools/file_operations.py

@@ -1,6 +1,8 @@
 # file_operations.py
 
 import os
+import shutil
+import subprocess
 import tempfile
 from typing import List
 
@@ -10,15 +12,14 @@ from pydantic_ai import RunContext
 # ---------------------------------------------------------------------------
 # Module-level helper functions (exposed for unit tests _and_ used as tools)
 # ---------------------------------------------------------------------------
-from code_puppy.messaging import (
-    emit_divider,
-    emit_error,
-    emit_info,
-    emit_success,
-    emit_system_message,
-    emit_warning,
+from code_puppy.messaging import (  # New structured messaging types
+    FileContentMessage,
+    FileEntry,
+    FileListingMessage,
+    GrepMatch,
+    GrepResultMessage,
+    get_message_bus,
 )
-from code_puppy.tools.common import generate_group_id
 
 
 # Pydantic models for tool return types
@@ -49,6 +50,7 @@ class MatchInfo(BaseModel):
 
 class GrepOutput(BaseModel):
     matches: List[MatchInfo]
+    error: str | None = None
 
 
 def is_likely_home_directory(directory):
@@ -107,54 +109,71 @@ def is_project_directory(directory):
         return False
 
 
+def would_match_directory(pattern: str, directory: str) -> bool:
+    """Check if a glob pattern would match the given directory path.
+
+    This is used to avoid adding ignore patterns that would inadvertently
+    exclude the directory we're actually trying to search in.
+
+    Args:
+        pattern: A glob pattern like '**/tmp/**' or 'node_modules'
+        directory: The directory path to check against
+
+    Returns:
+        True if the pattern would match the directory, False otherwise
+    """
+    import fnmatch
+
+    # Normalize the directory path
+    abs_dir = os.path.abspath(directory)
+    dir_name = os.path.basename(abs_dir)
+
+    # Strip leading/trailing wildcards and slashes for simpler matching
+    clean_pattern = pattern.strip("*").strip("/")
+
+    # Check if the directory name matches the pattern
+    if fnmatch.fnmatch(dir_name, clean_pattern):
+        return True
+
+    # Check if the full path contains the pattern
+    if fnmatch.fnmatch(abs_dir, pattern):
+        return True
+
+    # Check if any part of the path matches
+    path_parts = abs_dir.split(os.sep)
+    for part in path_parts:
+        if fnmatch.fnmatch(part, clean_pattern):
+            return True
+
+    return False
+
+
 def _list_files(
     context: RunContext, directory: str = ".", recursive: bool = True
 ) -> ListFileOutput:
-    import shutil
-    import subprocess
     import sys
 
     results = []
     directory = os.path.abspath(os.path.expanduser(directory))
 
-    # Build string representation
+    # Plain text output for LLM consumption
     output_lines = []
-
-    directory_listing_header = (
-        "\n[bold white on blue] DIRECTORY LISTING [/bold white on blue]"
-    )
-    output_lines.append(directory_listing_header)
-
-    directory_info = f"\U0001f4c2 [bold cyan]{directory}[/bold cyan] [dim](recursive={recursive})[/dim]\n"
-    output_lines.append(directory_info)
-
-    divider = "[dim]" + "─" * 100 + "\n" + "[/dim]"
-    output_lines.append(divider)
+    output_lines.append(f"DIRECTORY LISTING: {directory} (recursive={recursive})")
 
     if not os.path.exists(directory):
-        error_msg = (
-            f"[red bold]Error:[/red bold] Directory '{directory}' does not exist"
-        )
-        output_lines.append(error_msg)
-
-        output_lines.append(divider)
-        return ListFileOutput(content="\n".join(output_lines))
+        error_msg = f"Error: Directory '{directory}' does not exist"
+        return ListFileOutput(content=error_msg, error=error_msg)
     if not os.path.isdir(directory):
-        error_msg = f"[red bold]Error:[/red bold] '{directory}' is not a directory"
-        output_lines.append(error_msg)
-
-        output_lines.append(divider)
-        return ListFileOutput(content="\n".join(output_lines))
+        error_msg = f"Error: '{directory}' is not a directory"
+        return ListFileOutput(content=error_msg, error=error_msg)
 
     # Smart home directory detection - auto-limit recursion for performance
     # But allow recursion in tests (when context=None) or when explicitly requested
     if context is not None and is_likely_home_directory(directory) and recursive:
         if not is_project_directory(directory):
-            warning_msg = "[yellow bold]Warning:[/yellow bold] 🏠 Detected home directory - limiting to non-recursive listing for performance"
-            output_lines.append(warning_msg)
-
-            info_msg = f"[dim]💡 To force recursive listing in home directory, use list_files('{directory}', recursive=True) explicitly[/dim]"
-            output_lines.append(info_msg)
+            output_lines.append(
+                "Warning: Detected home directory - limiting to non-recursive listing for performance"
+            )
             recursive = False
 
     # Create a temporary ignore file with our ignore patterns
@@ -178,10 +197,10 @@ def _list_files(
                     rg_path = venv_rg_exe_path
                     break
 
-        if not rg_path:
-            error_msg = "[red bold]Error:[/red bold] ripgrep (rg) not found. Please install ripgrep to use this tool."
-            output_lines.append(error_msg)
-            return ListFileOutput(content="\n".join(output_lines))
+        if not rg_path and recursive:
+            # Only need ripgrep for recursive listings
+            error_msg = "Error: ripgrep (rg) not found. Please install ripgrep to use this tool."
+            return ListFileOutput(content=error_msg, error=error_msg)
 
         # Only use ripgrep for recursive listings
         if recursive:
@@ -198,6 +217,10 @@ def _list_files(
             ) as f:
                 ignore_file = f.name
                 for pattern in DIR_IGNORE_PATTERNS:
+                    # Skip patterns that would match the search directory itself
+                    # For example, if searching in /tmp/test-dir, skip **/tmp/**
+                    if would_match_directory(pattern, directory):
+                        continue
                     f.write(f"{pattern}\n")
 
             cmd.extend(["--ignore-file", ignore_file])
@@ -290,8 +313,6 @@ def _list_files(
         # ripgrep's --files option only returns files; we add directories and files ourselves
         if not recursive:
             try:
-                from code_puppy.tools.common import should_ignore_dir_path
-
                 entries = os.listdir(directory)
                 for entry in sorted(entries):
                     full_entry_path = os.path.join(directory, entry)
@@ -299,8 +320,9 @@ def _list_files(
                         continue
 
                     if os.path.isdir(full_entry_path):
-                        # Skip ignored directories
-                        if should_ignore_dir_path(full_entry_path):
+                        # In non-recursive mode, only skip obviously system/hidden directories
+                        # Don't use the full should_ignore_dir_path which is too aggressive
+                        if entry.startswith("."):
                             continue
                         results.append(
                             ListedFile(
@@ -330,17 +352,11 @@ def _list_files(
                 # Skip entries we can't access
                 pass
     except subprocess.TimeoutExpired:
-        error_msg = (
-            "[red bold]Error:[/red bold] List files command timed out after 30 seconds"
-        )
-        output_lines.append(error_msg)
-        return ListFileOutput(content="\n".join(output_lines))
+        error_msg = "Error: List files command timed out after 30 seconds"
+        return ListFileOutput(content=error_msg, error=error_msg)
     except Exception as e:
-        error_msg = (
-            f"[red bold]Error:[/red bold] Error during list files operation: {e}"
-        )
-        output_lines.append(error_msg)
-        return ListFileOutput(content="\n".join(output_lines))
+        error_msg = f"Error: Error during list files operation: {e}"
+        return ListFileOutput(content=error_msg, error=error_msg)
     finally:
         # Clean up the temporary ignore file
         if ignore_file and os.path.exists(ignore_file):
@@ -390,59 +406,48 @@ def _list_files(
     file_count = sum(1 for item in results if item.type == "file")
     total_size = sum(item.size for item in results if item.type == "file")
 
-    # Build the directory header section
-    dir_name = os.path.basename(directory) or directory
-    dir_header = f"\U0001f4c1 [bold blue]{dir_name}[/bold blue]"
-    output_lines.append(dir_header)
-
-    # Sort all items by path for consistent display
-    all_items = sorted(results, key=lambda x: x.path)
-
-    # Build file and directory tree representation
-    parent_dirs_with_content = set()
-    for item in all_items:
-        # Skip root directory entries with no path
+    # Build structured FileEntry objects for the UI
+    file_entries = []
+    for item in sorted(results, key=lambda x: x.path):
         if item.type == "directory" and not item.path:
             continue
+        file_entries.append(
+            FileEntry(
+                path=item.path,
+                type="dir" if item.type == "directory" else "file",
+                size=item.size,
+                depth=item.depth or 0,
+            )
+        )
 
-        # Track parent directories that contain files/dirs
-        if os.sep in item.path:
-            parent_path = os.path.dirname(item.path)
-            parent_dirs_with_content.add(parent_path)
-
-        # Calculate indentation depth based on path separators
-        depth = item.path.count(os.sep) + 1 if item.path else 0
-        prefix = ""
-        for d in range(depth):
-            if d == depth - 1:
-                prefix += "\u2514\u2500\u2500 "
-            else:
-                prefix += "    "
+    # Emit structured message for the UI
+    file_listing_msg = FileListingMessage(
+        directory=directory,
+        files=file_entries,
+        recursive=recursive,
+        total_size=total_size,
+        dir_count=dir_count,
+        file_count=file_count,
+    )
+    get_message_bus().emit(file_listing_msg)
 
-        # Get the display name (basename) of the item
+    # Build plain text output for LLM consumption
+    for item in sorted(results, key=lambda x: x.path):
+        if item.type == "directory" and not item.path:
+            continue
         name = os.path.basename(item.path) or item.path
-
-        # Add directory or file line with appropriate formatting
+        indent = "  " * (item.depth or 0)
         if item.type == "directory":
-            dir_line = f"{prefix}\U0001f4c1 [bold blue]{name}/[/bold blue]"
-            output_lines.append(dir_line)
+            output_lines.append(f"{indent}{name}/")
         else:
-            icon = get_file_icon(item.path)
             size_str = format_size(item.size)
-            file_line = f"{prefix}{icon} [green]{name}[/green] [dim]({size_str})[/dim]"
-            output_lines.append(file_line)
-
-    # Add summary information
-    summary_header = "\n[bold cyan]Summary:[/bold cyan]"
-    output_lines.append(summary_header)
-
-    summary_line = f"\U0001f4c1 [blue]{dir_count} directories[/blue], \U0001f4c4 [green]{file_count} files[/green] [dim]({format_size(total_size)} total)[/dim]"
-    output_lines.append(summary_line)
+            output_lines.append(f"{indent}{name} ({size_str})")
 
-    final_divider = "[dim]" + "─" * 100 + "\n" + "[/dim]"
-    output_lines.append(final_divider)
+    # Add summary
+    output_lines.append(
+        f"\nSummary: {dir_count} directories, {file_count} files ({format_size(total_size)} total)"
+    )
 
-    # Return the content string
     return ListFileOutput(content="\n".join(output_lines))
 
 
@@ -454,16 +459,6 @@ def _read_file(
 ) -> ReadFileOutput:
     file_path = os.path.abspath(os.path.expanduser(file_path))
 
-    # Generate group_id for this tool execution
-    group_id = generate_group_id("read_file", file_path)
-
-    # Build console message with optional parameters
-    console_msg = f"\n[bold white on blue] READ FILE [/bold white on blue] \U0001f4c2 [bold cyan]{file_path}[/bold cyan]"
-    if start_line is not None and num_lines is not None:
-        console_msg += f" [dim](lines {start_line}-{start_line + num_lines - 1})[/dim]"
-    emit_info(console_msg, message_group=group_id)
-
-    emit_divider(message_group=group_id)
     if not os.path.exists(file_path):
         error_msg = f"File {file_path} does not exist"
         return ReadFileOutput(content=error_msg, num_tokens=0, error=error_msg)
@@ -471,12 +466,15 @@ def _read_file(
         error_msg = f"{file_path} is not a file"
         return ReadFileOutput(content=error_msg, num_tokens=0, error=error_msg)
     try:
-        with open(file_path, "r", encoding="utf-8") as f:
+        # Use errors="surrogateescape" to handle files with invalid UTF-8 sequences
+        # This is common on Windows when files contain emojis or were created by
+        # applications that don't properly encode Unicode
+        with open(file_path, "r", encoding="utf-8", errors="surrogateescape") as f:
             if start_line is not None and num_lines is not None:
                 # Read only the specified lines
                 lines = f.readlines()
-                # Adjust for 1-based line numbering
-                start_idx = start_line - 1
+                # Adjust for 1-based line numbering and handle negative values
+                start_idx = start_line - 1 if start_line > 0 else 0
                 end_idx = start_idx + num_lines
                 # Ensure indices are within bounds
                 start_idx = max(0, start_idx)
@@ -486,6 +484,21 @@ def _read_file(
                 # Read the entire file
                 content = f.read()
 
+            # Sanitize the content to remove any surrogate characters that could
+            # cause issues when the content is later serialized or displayed
+            # This re-encodes with surrogatepass then decodes with replace to
+            # convert lone surrogates to replacement characters
+            try:
+                content = content.encode("utf-8", errors="surrogatepass").decode(
+                    "utf-8", errors="replace"
+                )
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                # If that fails, do a more aggressive cleanup
+                content = "".join(
+                    char if ord(char) < 0xD800 or ord(char) > 0xDFFF else "\ufffd"
+                    for char in content
+                )
+
             # Simple approximation: ~4 characters per token
             num_tokens = len(content) // 4
             if num_tokens > 10000:
@@ -494,6 +507,30 @@ def _read_file(
                     error="The file is massive, greater than 10,000 tokens which is dangerous to read entirely. Please read this file in chunks.",
                     num_tokens=0,
                 )
+
+            # Count total lines for the message
+            total_lines = content.count("\n") + (
+                1 if content and not content.endswith("\n") else 0
+            )
+
+            # Emit structured message for the UI
+            # Only include start_line/num_lines if they are valid positive integers
+            emit_start_line = (
+                start_line if start_line is not None and start_line >= 1 else None
+            )
+            emit_num_lines = (
+                num_lines if num_lines is not None and num_lines >= 1 else None
+            )
+            file_content_msg = FileContentMessage(
+                path=file_path,
+                content=content,
+                start_line=emit_start_line,
+                num_lines=emit_num_lines,
+                total_lines=total_lines,
+                num_tokens=num_tokens,
+            )
+            get_message_bus().emit(file_content_msg)
+
         return ReadFileOutput(content=content, num_tokens=num_tokens)
     except (FileNotFoundError, PermissionError):
         # For backward compatibility with tests, return "FILE NOT FOUND" for these specific errors
@@ -504,6 +541,33 @@ def _read_file(
         return ReadFileOutput(content=message, num_tokens=0, error=message)
 
 
+def _sanitize_string(text: str) -> str:
+    """Sanitize a string to remove invalid Unicode surrogates.
+
+    This handles encoding issues common on Windows with copy-paste operations.
+    """
+    if not text:
+        return text
+    try:
+        # Try encoding - if it works, string is clean
+        text.encode("utf-8")
+        return text
+    except UnicodeEncodeError:
+        pass
+
+    try:
+        # Encode allowing surrogates, then decode replacing them
+        return text.encode("utf-8", errors="surrogatepass").decode(
+            "utf-8", errors="replace"
+        )
+    except (UnicodeEncodeError, UnicodeDecodeError):
+        # Last resort: filter out surrogate characters
+        return "".join(
+            char if ord(char) < 0xD800 or ord(char) > 0xDFFF else "\ufffd"
+            for char in text
+        )
+
+
 def _grep(context: RunContext, search_string: str, directory: str = ".") -> GrepOutput:
     import json
     import os
@@ -511,17 +575,12 @@ def _grep(context: RunContext, search_string: str, directory: str = ".") -> Grep
     import subprocess
     import sys
 
+    # Sanitize search string to handle any surrogates from copy-paste
+    search_string = _sanitize_string(search_string)
+
     directory = os.path.abspath(os.path.expanduser(directory))
     matches: List[MatchInfo] = []
-
-    # Generate group_id for this tool execution
-    group_id = generate_group_id("grep", f"{directory}_{search_string}")
-
-    emit_info(
-        f"\n[bold white on blue] GREP [/bold white on blue] \U0001f4c2 [bold cyan]{directory}[/bold cyan] [dim]for '{search_string}'[/dim]",
-        message_group=group_id,
-    )
-    emit_divider(message_group=group_id)
+    error_message: str | None = None
 
     # Create a temporary ignore file with our ignore patterns
     ignore_file = None
@@ -553,11 +612,10 @@ def _grep(context: RunContext, search_string: str, directory: str = ".") -> Grep
                     break
 
         if not rg_path:
-            emit_error(
-                "ripgrep (rg) not found. Please install ripgrep to use this tool.",
-                message_group=group_id,
+            error_message = (
+                "ripgrep (rg) not found. Please install ripgrep to use this tool."
             )
-            return GrepOutput(matches=[])
+            return GrepOutput(matches=[], error=error_message)
 
         cmd = [
             rg_path,
@@ -579,7 +637,15 @@ def _grep(context: RunContext, search_string: str, directory: str = ".") -> Grep
 
         cmd.extend(["--ignore-file", ignore_file])
         cmd.extend([search_string, directory])
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+        # Use encoding with error handling to handle files with invalid UTF-8
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=30,
+            encoding="utf-8",
+            errors="replace",  # Replace invalid chars instead of crashing
+        )
 
         # Parse the JSON output from ripgrep
         for line in result.stdout.strip().split("\n"):
@@ -603,49 +669,57 @@ def _grep(context: RunContext, search_string: str, directory: str = ".") -> Grep
                     if len(line_content.strip()) > 512:
                         line_content = line_content.strip()[0:512]
                     if file_path and line_number:
+                        # Sanitize content to handle any remaining encoding issues
                         match_info = MatchInfo(
-                            file_path=file_path,
+                            file_path=_sanitize_string(file_path),
                             line_number=line_number,
-                            line_content=line_content.strip(),
+                            line_content=_sanitize_string(line_content.strip()),
                         )
                         matches.append(match_info)
                         # Limit to 50 matches total, same as original implementation
                         if len(matches) >= 50:
                             break
-                        emit_system_message(
-                            f"[green]Match:[/green] {file_path}:{line_number} - {line_content.strip()}",
-                            message_group=group_id,
-                        )
             except json.JSONDecodeError:
                 # Skip lines that aren't valid JSON
                 continue
 
-        if not matches:
-            emit_warning(
-                f"No matches found for '{search_string}' in {directory}",
-                message_group=group_id,
-            )
-        else:
-            emit_success(
-                f"Found {len(matches)} match(es) for '{search_string}' in {directory}",
-                message_group=group_id,
-            )
-
     except subprocess.TimeoutExpired:
-        emit_error("Grep command timed out after 30 seconds", message_group=group_id)
+        error_message = "Grep command timed out after 30 seconds"
     except FileNotFoundError:
-        emit_error(
-            "ripgrep (rg) not found. Please install ripgrep to use this tool.",
-            message_group=group_id,
+        error_message = (
+            "ripgrep (rg) not found. Please install ripgrep to use this tool."
         )
     except Exception as e:
-        emit_error(f"Error during grep operation: {e}", message_group=group_id)
+        error_message = f"Error during grep operation: {e}"
     finally:
         # Clean up the temporary ignore file
         if ignore_file and os.path.exists(ignore_file):
             os.unlink(ignore_file)
 
-    return GrepOutput(matches=matches)
+    # Build structured GrepMatch objects for the UI
+    grep_matches = [
+        GrepMatch(
+            file_path=m.file_path or "",
+            line_number=m.line_number or 1,
+            line_content=m.line_content or "",
+        )
+        for m in matches
+    ]
+
+    # Count unique files searched (approximation based on matches)
+    unique_files = len(set(m.file_path for m in matches)) if matches else 0
+
+    # Emit structured message for the UI (only once, at the end)
+    grep_result_msg = GrepResultMessage(
+        search_term=search_string,
+        directory=directory,
+        matches=grep_matches,
+        total_matches=len(matches),
+        files_searched=unique_files,
+    )
+    get_message_bus().emit(grep_result_msg)
+
+    return GrepOutput(matches=matches, error=error_message)
 
 
 def register_list_files(agent):
@@ -706,10 +780,8 @@ def register_list_files(agent):
             recursive = False
         result = _list_files(context, directory, recursive)
 
-        # Emit the content directly to ensure it's displayed to the user
-        emit_info(
-            result.content, message_group=generate_group_id("list_files", directory)
-        )
+        # The structured FileListingMessage is already emitted by _list_files
+        # No need to emit again here
         if warning:
             result.error = warning
         if (len(result.content)) > 200000: