ripperdoc 0.2.9__py3-none-any.whl → 0.3.0__py3-none-any.whl

ripperdoc/tools/file_read_tool.py

@@ -5,8 +5,9 @@ Allows the AI to read file contents.
 
 import os
 from pathlib import Path
-from typing import AsyncGenerator, List, Optional
+from typing import AsyncGenerator, List, Optional, Tuple
 from pydantic import BaseModel, Field
+from charset_normalizer import from_bytes
 
 from ripperdoc.core.tool import (
     Tool,
@@ -23,6 +24,105 @@ from ripperdoc.utils.path_ignore import check_path_for_tool
 logger = get_logger()
 
 
+def detect_file_encoding(file_path: str) -> Tuple[Optional[str], float]:
+    """Detect file encoding using charset-normalizer.
+
+    Returns:
+        Tuple of (encoding, confidence). encoding is None if detection failed.
+    """
+    try:
+        with open(file_path, "rb") as f:
+            raw_data = f.read()
+        results = from_bytes(raw_data)
+
+        if not results:
+            return None, 0.0
+
+        best = results.best()
+        if not best:
+            return None, 0.0
+
+        # For Chinese content, prefer GB encodings over Big5/others
+        # charset-normalizer sometimes picks Big5 for simplified Chinese
+        if best.language == "Chinese":
+            gb_encodings = {"gb18030", "gbk", "gb2312"}
+            for result in results:
+                if result.encoding.lower() in gb_encodings:
+                    return result.encoding, 0.9
+
+        return best.encoding, 0.9
+    except (OSError, IOError) as e:
+        logger.warning("Failed to detect encoding for %s: %s", file_path, e)
+        return None, 0.0
+
+
+def read_file_with_encoding(file_path: str) -> Tuple[Optional[List[str]], str, Optional[str]]:
+    """Read file with proper encoding detection.
+
+    Returns:
+        Tuple of (lines, encoding_used, error_message).
+        If successful: (lines, encoding, None)
+        If failed: (None, "", error_message)
+    """
+    # First, try UTF-8 (most common)
+    try:
+        with open(file_path, "r", encoding="utf-8", errors="strict") as f:
+            lines = f.readlines()
+        return lines, "utf-8", None
+    except UnicodeDecodeError:
+        pass
+
+    # UTF-8 failed, use charset-normalizer to detect encoding
+    detected_encoding, confidence = detect_file_encoding(file_path)
+
+    if detected_encoding:
+        try:
+            with open(file_path, "r", encoding=detected_encoding, errors="strict") as f:
+                lines = f.readlines()
+            logger.info(
+                "File %s decoded using detected encoding %s",
+                file_path,
+                detected_encoding,
+            )
+            return lines, detected_encoding, None
+        except (UnicodeDecodeError, LookupError) as e:
+            logger.warning(
+                "Failed to read %s with detected encoding %s: %s",
+                file_path,
+                detected_encoding,
+                e,
+            )
+
+    # Detection failed - try latin-1 as last resort (can decode any byte sequence)
+    try:
+        with open(file_path, "r", encoding="latin-1", errors="strict") as f:
+            lines = f.readlines()
+        logger.warning(
+            "File %s: encoding detection failed, using latin-1 fallback",
+            file_path,
+        )
+        return lines, "latin-1", None
+    except (UnicodeDecodeError, LookupError):
+        pass
+
+    # All attempts failed - return error
+    error_msg = (
+        f"Unable to determine file encoding. "
+        f"Detected: {detected_encoding or 'unknown'} (confidence: {confidence * 100:.0f}%). "
+        f"Tried fallback encodings: utf-8, latin-1. "
+        f"Please convert the file to UTF-8."
+    )
+    return None, "", error_msg
+
+
+# Maximum file size to read (default 256KB)
+# Can be overridden via env var in bytes
+MAX_FILE_SIZE_BYTES = int(os.getenv("RIPPERDOC_MAX_READ_FILE_SIZE_BYTES", "262144"))  # 256KB
+
+# Maximum lines to read when no limit is specified (default 2000 lines)
+MAX_READ_LINES = int(os.getenv("RIPPERDOC_MAX_READ_LINES", "2000"))
+
+
 class FileReadToolInput(BaseModel):
     """Input schema for FileReadTool."""
 
@@ -75,6 +175,7 @@ and limit to read only a portion of the file."""
             "Read a file from the local filesystem.\n\n"
             "Usage:\n"
             "- The file_path parameter must be an absolute path (not relative).\n"
+            "- Files larger than 256KB or with more than 2000 lines require using offset and limit parameters.\n"
             "- By default, the entire file is read. You can optionally specify a line offset and limit (handy for long files); offset is zero-based and output line numbers start at 1.\n"
             "- Lines longer than 2000 characters are truncated in the output.\n"
             "- Results are returned with cat -n style numbering: spaces + line number + tab, then the file content.\n"
@@ -140,11 +241,60 @@ and limit to read only a portion of the file."""
         """Read the file."""
 
         try:
-            with open(input_data.file_path, "r", encoding="utf-8", errors="replace") as f:
-                lines = f.readlines()
+            # Check file size before reading to prevent memory exhaustion
+            file_size = os.path.getsize(input_data.file_path)
+            if file_size > MAX_FILE_SIZE_BYTES:
+                size_kb = file_size / 1024
+                limit_kb = MAX_FILE_SIZE_BYTES / 1024
+                error_output = FileReadToolOutput(
+                    content=f"File too large to read: {size_kb:.1f}KB exceeds limit of {limit_kb:.0f}KB. Use offset and limit parameters to read portions.",
+                    file_path=input_data.file_path,
+                    line_count=0,
+                    offset=0,
+                    limit=None,
+                )
+                yield ToolResult(
+                    data=error_output,
+                    result_for_assistant=f"Error: File {input_data.file_path} is too large ({size_kb:.1f}KB). Maximum size is {limit_kb:.0f}KB. Use offset and limit to read portions, e.g., Read(file_path='{input_data.file_path}', offset=0, limit=500).",
+                )
+                return
+
+            # Detect and read file with proper encoding
+            lines, used_encoding, encoding_error = read_file_with_encoding(input_data.file_path)
+
+            if lines is None:
+                # Encoding detection failed - return warning to LLM
+                error_output = FileReadToolOutput(
+                    content=f"Encoding error: {encoding_error}",
+                    file_path=input_data.file_path,
+                    line_count=0,
+                    offset=0,
+                    limit=None,
+                )
+                yield ToolResult(
+                    data=error_output,
+                    result_for_assistant=f"Error: Cannot read file {input_data.file_path}. {encoding_error}",
+                )
+                return
 
             offset = input_data.offset or 0
             limit = input_data.limit
+            total_lines = len(lines)
+
+            # Check line count if no limit is specified (to prevent context overflow)
+            if limit is None and total_lines > MAX_READ_LINES:
+                error_output = FileReadToolOutput(
+                    content=f"File too large: {total_lines} lines exceeds limit of {MAX_READ_LINES} lines. Use offset and limit parameters to read portions.",
+                    file_path=input_data.file_path,
+                    line_count=total_lines,
+                    offset=0,
+                    limit=None,
+                )
+                yield ToolResult(
+                    data=error_output,
+                    result_for_assistant=f"Error: File {input_data.file_path} has {total_lines} lines, exceeding the limit of {MAX_READ_LINES} lines when reading without limit parameter. Use offset and limit to read portions, e.g., Read(file_path='{input_data.file_path}', offset=0, limit=500).",
+                )
+                return
 
             # Apply offset and limit
             if limit is not None:
@@ -164,6 +314,7 @@ and limit to read only a portion of the file."""
                     getattr(context, "file_state_cache", {}),
                     offset=offset,
                     limit=limit,
+                    encoding=used_encoding,
                 )
             except (OSError, IOError, RuntimeError) as exc:
                 logger.warning(

ripperdoc/tools/file_write_tool.py

@@ -19,10 +19,48 @@ from ripperdoc.core.tool import (
 from ripperdoc.utils.log import get_logger
 from ripperdoc.utils.file_watch import record_snapshot
 from ripperdoc.utils.path_ignore import check_path_for_tool
+from ripperdoc.tools.file_read_tool import detect_file_encoding
 
 logger = get_logger()
 
 
+def determine_write_encoding(file_path: str, content: str) -> str:
+    """Determine the best encoding to use for writing a file.
+
+    Strategy:
+    1. If file doesn't exist -> use UTF-8
+    2. If file exists -> detect its encoding using charset-normalizer
+    3. Verify content can be encoded with target encoding
+    4. If encoding fails (e.g., emoji in GBK) -> fall back to UTF-8
+
+    Returns:
+        The encoding to use for writing.
+    """
+    # Default to UTF-8 for new files
+    if not os.path.exists(file_path):
+        return "utf-8"
+
+    # Detect existing file's encoding
+    detected_encoding, _ = detect_file_encoding(file_path)
+
+    # If detection failed, use UTF-8
+    if not detected_encoding:
+        return "utf-8"
+
+    # Verify content can be encoded with detected encoding
+    try:
+        content.encode(detected_encoding)
+        return detected_encoding
+    except (UnicodeEncodeError, LookupError):
+        # Content can't be encoded (e.g., emoji in GBK), fall back to UTF-8
+        logger.info(
+            "Content cannot be encoded with %s, falling back to UTF-8 for %s",
+            detected_encoding,
+            file_path,
+        )
+        return "utf-8"
+
+
 class FileWriteToolInput(BaseModel):
     """Input schema for FileWriteTool."""
 
@@ -104,6 +142,13 @@ NEVER write new files unless explicitly required by the user."""
 
         file_path = os.path.abspath(input_data.file_path)
 
+        file_path_obj = Path(file_path)
+        should_proceed, warning_msg = check_path_for_tool(
+            file_path_obj, tool_name="Write", warn_only=True
+        )
+        if warning_msg:
+            logger.warning("[file_write_tool] %s", warning_msg)
+
         # If file doesn't exist, it's a new file - allow without reading first
         if not os.path.exists(file_path):
             return ValidationResult(result=True)
@@ -132,14 +177,6 @@ NEVER write new files unless explicitly required by the user."""
         except OSError:
             pass  # File mtime check failed, proceed anyway
 
-        # Check if path is ignored (warning for write operations)
-        file_path_obj = Path(file_path)
-        should_proceed, warning_msg = check_path_for_tool(
-            file_path_obj, tool_name="Write", warn_only=True
-        )
-        if warning_msg:
-            logger.warning("[file_write_tool] %s", warning_msg)
-
         return ValidationResult(result=True)
 
     def render_result_for_assistant(self, output: FileWriteToolOutput) -> str:
@@ -156,11 +193,15 @@ NEVER write new files unless explicitly required by the user."""
         """Write the file."""
 
         try:
-            # Write the file
-            with open(input_data.file_path, "w", encoding="utf-8") as f:
+            # Determine encoding based on target file and content
+            file_path = os.path.abspath(input_data.file_path)
+            encoding = determine_write_encoding(file_path, input_data.content)
+
+            # Write the file with the appropriate encoding
+            with open(input_data.file_path, "w", encoding=encoding) as f:
                 f.write(input_data.content)
 
-            bytes_written = len(input_data.content.encode("utf-8"))
+            bytes_written = len(input_data.content.encode(encoding))
 
             # Use absolute path to ensure consistency with validation lookup
             abs_file_path = os.path.abspath(input_data.file_path)
@@ -169,6 +210,7 @@ NEVER write new files unless explicitly required by the user."""
                     abs_file_path,
                     input_data.content,
                     getattr(context, "file_state_cache", {}),
+                    encoding=encoding,
                 )
             except (OSError, IOError, RuntimeError) as exc:
                 logger.warning(

ripperdoc/tools/grep_tool.py

@@ -78,6 +78,35 @@ def _normalize_glob_for_grep(glob_pattern: str) -> str:
     return glob_pattern.split("/")[-1] or glob_pattern
 
 
+_GREP_SUPPORTS_PCRE: Optional[bool] = None
+
+
+def _grep_supports_pcre() -> bool:
+    """Detect if the system grep supports -P (Perl regex), caching the result."""
+    global _GREP_SUPPORTS_PCRE
+    if _GREP_SUPPORTS_PCRE is not None:
+        return _GREP_SUPPORTS_PCRE
+
+    if shutil.which("grep") is None:
+        _GREP_SUPPORTS_PCRE = False
+        return _GREP_SUPPORTS_PCRE
+
+    try:
+        proc = subprocess.run(
+            ["grep", "-P", ""],
+            stdin=subprocess.DEVNULL,  # Fix: prevent waiting for stdin
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            check=False,
+            timeout=15,  # Safety timeout
+        )
+        _GREP_SUPPORTS_PCRE = proc.returncode in (0, 1)
+    except (OSError, ValueError, subprocess.SubprocessError, subprocess.TimeoutExpired):
+        _GREP_SUPPORTS_PCRE = False
+
+    return _GREP_SUPPORTS_PCRE
+
+
 class GrepToolInput(BaseModel):
     """Input schema for GrepTool."""
 
@@ -234,11 +263,36 @@ class GrepTool(Tool[GrepToolInput, GrepToolOutput]):
         self, input_data: GrepToolInput, _context: ToolUseContext
     ) -> AsyncGenerator[ToolOutput, None]:
         """Search for the pattern."""
+        logger.debug(
+            "[grep_tool] call ENTER: pattern='%s' path='%s'", input_data.pattern, input_data.path
+        )
 
         try:
             search_path = input_data.path or "."
 
+            async def _run_search(command: List[str]) -> Tuple[int, str, str]:
+                """Execute the search command and return decoded output."""
+                logger.debug(
+                    "[grep_tool] _run_search: BEFORE create_subprocess_exec, cmd=%s", command[:5]
+                )
+                process = await asyncio.create_subprocess_exec(
+                    *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+                )
+                logger.debug(
+                    "[grep_tool] _run_search: AFTER create_subprocess_exec, pid=%s", process.pid
+                )
+                logger.debug("[grep_tool] _run_search: BEFORE communicate()")
+                stdout, stderr = await process.communicate()
+                logger.debug(
+                    "[grep_tool] _run_search: AFTER communicate(), returncode=%s",
+                    process.returncode,
+                )
+                stdout_text = stdout.decode("utf-8", errors="ignore") if stdout else ""
+                stderr_text = stderr.decode("utf-8", errors="ignore") if stderr else ""
+                return process.returncode or 0, stdout_text, stderr_text
+
             use_ripgrep = shutil.which("rg") is not None
+            logger.debug("[grep_tool] use_ripgrep=%s", use_ripgrep)
             pattern = input_data.pattern
 
             if use_ripgrep:
@@ -263,7 +317,11 @@ class GrepTool(Tool[GrepToolInput, GrepToolOutput]):
                 cmd.append(search_path)
             else:
                 # Fallback to grep (note: grep --include matches basenames only)
-                cmd = ["grep", "-r", "--color=never", "-P"]
+                logger.debug("[grep_tool] Using grep fallback, checking PCRE support...")
+                use_pcre = _grep_supports_pcre()
+                logger.debug("[grep_tool] PCRE support check done: use_pcre=%s", use_pcre)
+                cmd = ["grep", "-r", "--color=never", "-P" if use_pcre else "-E"]
+                logger.debug("[grep_tool] Building grep command...")
 
                 if input_data.case_insensitive:
                     cmd.append("-i")
@@ -285,20 +343,52 @@ class GrepTool(Tool[GrepToolInput, GrepToolOutput]):
 
                 cmd.append(search_path)
 
-            # Run grep asynchronously
-            process = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            logger.debug("[grep_tool] BEFORE _run_search, cmd=%s", cmd)
+            returncode, stdout_text, stderr_text = await _run_search(cmd)
+            logger.debug(
+                "[grep_tool] AFTER _run_search, returncode=%s, stdout_len=%d",
+                returncode,
+                len(stdout_text),
             )
-
-            stdout, stderr = await process.communicate()
-            returncode = process.returncode
+            fallback_attempted = False
+
+            if returncode not in (0, 1):
+                if not use_ripgrep and "-P" in cmd:
+                    # BSD grep lacks -P; retry with extended regex before surfacing the error.
+                    fallback_attempted = True
+                    cmd = [flag if flag != "-P" else "-E" for flag in cmd]
+                    returncode, stdout_text, stderr_text = await _run_search(cmd)
+
+                if returncode not in (0, 1):
+                    error_msg = stderr_text.strip() or f"grep exited with status {returncode}"
+                    logger.warning(
+                        "[grep_tool] Grep command failed",
+                        extra={
+                            "pattern": input_data.pattern,
+                            "path": input_data.path,
+                            "returncode": returncode,
+                            "stderr": error_msg,
+                            "fallback_to_E": fallback_attempted,
+                        },
+                    )
+                    error_output = GrepToolOutput(
+                        matches=[],
+                        pattern=input_data.pattern,
+                        total_files=0,
+                        total_matches=0,
+                        output_mode=input_data.output_mode,
+                        head_limit=input_data.head_limit,
+                    )
+                    yield ToolResult(
+                        data=error_output, result_for_assistant=f"Grep error: {error_msg}"
+                    )
+                    return
 
             # Parse output
             matches: List[GrepMatch] = []
             total_matches = 0
             total_files = 0
             omitted_results = 0
-            stdout_text = stdout.decode("utf-8", errors="ignore") if stdout else ""
             lines = [line for line in stdout_text.split("\n") if line]
 
             if returncode in (0, 1):  # 0 = matches found, 1 = no matches (ripgrep/grep)