ripperdoc 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

ripperdoc/tools/file_edit_tool.py

@@ -5,7 +5,6 @@ Allows the AI to edit files by replacing text.
 
 import contextlib
 import os
-import tempfile
 from pathlib import Path
 from typing import AsyncGenerator, Generator, List, Optional, TextIO
 from pydantic import BaseModel, Field
@@ -20,8 +19,14 @@ from ripperdoc.core.tool import (
 )
 from ripperdoc.utils.log import get_logger
 from ripperdoc.utils.platform import HAS_FCNTL
-from ripperdoc.utils.file_watch import record_snapshot
 from ripperdoc.utils.path_ignore import check_path_for_tool
+from ripperdoc.utils.file_editing import (
+    atomic_write_with_fallback,
+    file_lock,
+    open_locked_file,
+    safe_record_snapshot,
+    select_write_encoding,
+)
 from ripperdoc.tools.file_read_tool import detect_file_encoding
 
 logger = get_logger()
@@ -38,44 +43,22 @@ def determine_edit_encoding(file_path: str, new_content: str) -> str:
     if not detected_encoding:
         return "utf-8"
 
-    # Verify new content can be encoded
-    try:
-        new_content.encode(detected_encoding)
-        return detected_encoding
-    except (UnicodeEncodeError, LookupError):
-        logger.info(
-            "New content cannot be encoded with %s, falling back to UTF-8 for %s",
-            detected_encoding,
-            file_path,
-        )
-        return "utf-8"
+    return select_write_encoding(
+        detected_encoding,
+        new_content,
+        file_path,
+        log_prefix="[file_edit_tool]",
+    )
 
 
 @contextlib.contextmanager
 def _file_lock(file_handle: TextIO, exclusive: bool = True) -> Generator[None, None, None]:
-    """Acquire a file lock, with fallback for systems without fcntl.
-
-    Args:
-        file_handle: An open file handle to lock
-        exclusive: If True, acquire exclusive lock; otherwise shared lock
-
-    Yields:
-        None
-    """
+    """Compatibility wrapper for tests that expect _file_lock in this module."""
     if not HAS_FCNTL:
-        # On Windows or systems without fcntl, skip locking
         yield
         return
-
-    import fcntl
-
-    lock_type = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH
-    try:
-        fcntl.flock(file_handle.fileno(), lock_type)
+    with file_lock(file_handle, exclusive=exclusive):
         yield
-    finally:
-        with contextlib.suppress(OSError):
-            fcntl.flock(file_handle.fileno(), fcntl.LOCK_UN)
 
 
 class FileEditToolInput(BaseModel):
@@ -247,71 +230,20 @@ match exactly (including whitespace and indentation)."""
             file_encoding = "utf-8"
 
         try:
-            # Open file with exclusive lock to prevent concurrent modifications
-            # Use r+ mode to get a file handle we can lock before reading
-            #
-            # TOCTOU mitigation strategy:
-            # 1. Record mtime immediately after open (pre_lock_mtime)
-            # 2. Acquire exclusive lock
-            # 3. Check mtime again after lock (post_lock_mtime)
-            # 4. If pre != post, file was modified in the window between open and lock
-            # 5. Also validate against cached snapshot timestamp
-            with open(abs_file_path, "r+", encoding=file_encoding) as f:
-                # Record mtime immediately after open, before acquiring lock
-                try:
-                    pre_lock_mtime = os.fstat(f.fileno()).st_mtime
-                except OSError:
-                    pre_lock_mtime = None
-
-                with _file_lock(f, exclusive=True):
-                    # Check mtime after acquiring lock to detect modifications
-                    # during the window between open() and lock acquisition
-                    try:
-                        post_lock_mtime = os.fstat(f.fileno()).st_mtime
-                    except OSError:
-                        post_lock_mtime = None
-
-                    # Detect modification during open->lock window
-                    if pre_lock_mtime is not None and post_lock_mtime is not None:
-                        if post_lock_mtime > pre_lock_mtime:
-                            output = FileEditToolOutput(
-                                file_path=input_data.file_path,
-                                replacements_made=0,
-                                success=False,
-                                message="File was modified while acquiring lock. Please retry.",
-                            )
-                            yield ToolResult(
-                                data=output,
-                                result_for_assistant=self.render_result_for_assistant(output),
-                            )
-                            return
-
-                    # Validate against cached snapshot timestamp
-                    if file_snapshot and post_lock_mtime is not None:
-                        if post_lock_mtime > file_snapshot.timestamp:
-                            output = FileEditToolOutput(
-                                file_path=input_data.file_path,
-                                replacements_made=0,
-                                success=False,
-                                message="File has been modified since read, either by the user "
-                                "or by a linter. Read it again before attempting to edit it.",
-                            )
-                            yield ToolResult(
-                                data=output,
-                                result_for_assistant=self.render_result_for_assistant(output),
-                            )
-                            return
-
-                    # Read content while holding the lock
-                    content = f.read()
-
-                    # Check if old_string exists
-                    if input_data.old_string not in content:
+            # Open file with exclusive lock to prevent concurrent modifications.
+            # Uses shared helper for consistent TOCTOU protection.
+            with open_locked_file(abs_file_path, file_encoding) as (
+                f,
+                pre_lock_mtime,
+                post_lock_mtime,
+            ):
+                if pre_lock_mtime is not None and post_lock_mtime is not None:
+                    if post_lock_mtime > pre_lock_mtime:
                         output = FileEditToolOutput(
                             file_path=input_data.file_path,
                             replacements_made=0,
                             success=False,
-                            message=f"String not found in file: {input_data.file_path}",
+                            message="File was modified while acquiring lock. Please retry.",
                         )
                         yield ToolResult(
                             data=output,
@@ -319,17 +251,14 @@ match exactly (including whitespace and indentation)."""
                         )
                         return
 
-                    # Count occurrences
-                    occurrence_count = content.count(input_data.old_string)
-
-                    # Check for ambiguity if not replace_all
-                    if not input_data.replace_all and occurrence_count > 1:
+                if file_snapshot and post_lock_mtime is not None:
+                    if post_lock_mtime > file_snapshot.timestamp:
                         output = FileEditToolOutput(
                             file_path=input_data.file_path,
                             replacements_made=0,
                             success=False,
-                            message=f"String appears {occurrence_count} times in file. "
-                            f"Either provide a unique string or use replace_all=true",
+                            message="File has been modified since read, either by the user "
+                            "or by a linter. Read it again before attempting to edit it.",
                         )
                         yield ToolResult(
                             data=output,
@@ -337,91 +266,81 @@ match exactly (including whitespace and indentation)."""
                         )
                         return
 
-                    # Perform replacement
-                    if input_data.replace_all:
-                        new_content = content.replace(input_data.old_string, input_data.new_string)
-                        replacements = occurrence_count
-                    else:
-                        new_content = content.replace(
-                            input_data.old_string, input_data.new_string, 1
-                        )
-                        replacements = 1
-
-                    # Verify new content can be encoded with file's encoding
-                    # If not, fall back to UTF-8
-                    write_encoding = file_encoding
-                    try:
-                        new_content.encode(file_encoding)
-                    except (UnicodeEncodeError, LookupError):
-                        logger.info(
-                            "New content cannot be encoded with %s, using UTF-8 for %s",
-                            file_encoding,
-                            abs_file_path,
-                        )
-                        write_encoding = "utf-8"
-
-                    # Atomic write: write to temp file then rename
-                    # This ensures the file is either fully written or not at all
-                    file_dir = os.path.dirname(abs_file_path)
-                    try:
-                        # Create temp file in same directory to ensure same filesystem
-                        fd, temp_path = tempfile.mkstemp(
-                            dir=file_dir, prefix=".ripperdoc_edit_", suffix=".tmp"
-                        )
-                        try:
-                            with os.fdopen(fd, "w", encoding=write_encoding) as temp_f:
-                                temp_f.write(new_content)
-                            # Preserve original file permissions
-                            original_stat = os.fstat(f.fileno())
-                            os.chmod(temp_path, original_stat.st_mode)
-                            # Atomic replace (works on Unix, best-effort on Windows)
-                            os.replace(temp_path, abs_file_path)
-                        except Exception:
-                            # Clean up temp file on failure
-                            with contextlib.suppress(OSError):
-                                os.unlink(temp_path)
-                            raise
-                    except OSError as atomic_error:
-                        # Fallback to in-place write if atomic write fails
-                        # (e.g., cross-filesystem issues)
-                        # Re-verify file hasn't changed before fallback write (TOCTOU protection)
-                        f.seek(0)
-                        current_content = f.read()
-                        if current_content != content:
-                            output = FileEditToolOutput(
-                                file_path=input_data.file_path,
-                                replacements_made=0,
-                                success=False,
-                                message="File was modified during atomic write fallback. Please retry.",
-                            )
-                            yield ToolResult(
-                                data=output,
-                                result_for_assistant=self.render_result_for_assistant(output),
-                            )
-                            return
-                        f.seek(0)
-                        f.truncate()
-                        f.write(new_content)
-                        logger.debug(
-                            "[file_edit_tool] Atomic write failed, used fallback: %s",
-                            atomic_error,
-                        )
+                content = f.read()
+
+                if input_data.old_string not in content:
+                    output = FileEditToolOutput(
+                        file_path=input_data.file_path,
+                        replacements_made=0,
+                        success=False,
+                        message=f"String not found in file: {input_data.file_path}",
+                    )
+                    yield ToolResult(
+                        data=output,
+                        result_for_assistant=self.render_result_for_assistant(output),
+                    )
+                    return
+
+                occurrence_count = content.count(input_data.old_string)
+
+                if not input_data.replace_all and occurrence_count > 1:
+                    output = FileEditToolOutput(
+                        file_path=input_data.file_path,
+                        replacements_made=0,
+                        success=False,
+                        message=f"String appears {occurrence_count} times in file. "
+                        f"Either provide a unique string or use replace_all=true",
+                    )
+                    yield ToolResult(
+                        data=output,
+                        result_for_assistant=self.render_result_for_assistant(output),
+                    )
+                    return
+
+                if input_data.replace_all:
+                    new_content = content.replace(input_data.old_string, input_data.new_string)
+                    replacements = occurrence_count
+                else:
+                    new_content = content.replace(input_data.old_string, input_data.new_string, 1)
+                    replacements = 1
+
+                write_encoding = select_write_encoding(
+                    file_encoding,
+                    new_content,
+                    abs_file_path,
+                    log_prefix="[file_edit_tool]",
+                )
 
-            # Record the new snapshot after successful edit
-            try:
-                record_snapshot(
+                write_error = atomic_write_with_fallback(
+                    f,
                     abs_file_path,
                     new_content,
-                    getattr(context, "file_state_cache", {}),
-                    encoding=write_encoding,
-                )
-            except (OSError, IOError, RuntimeError) as exc:
-                logger.warning(
-                    "[file_edit_tool] Failed to record file snapshot: %s: %s",
-                    type(exc).__name__,
-                    exc,
-                    extra={"file_path": abs_file_path},
+                    write_encoding,
+                    content,
+                    temp_prefix=".ripperdoc_edit_",
+                    log_prefix="[file_edit_tool]",
+                    conflict_message="File was modified during atomic write fallback. Please retry.",
                 )
+                if write_error:
+                    output = FileEditToolOutput(
+                        file_path=input_data.file_path,
+                        replacements_made=0,
+                        success=False,
+                        message=write_error,
+                    )
+                    yield ToolResult(
+                        data=output,
+                        result_for_assistant=self.render_result_for_assistant(output),
+                    )
+                    return
+
+            safe_record_snapshot(
+                abs_file_path,
+                new_content,
+                getattr(context, "file_state_cache", {}),
+                encoding=write_encoding,
+                log_prefix="[file_edit_tool]",
+            )
 
             # Generate diff for display
             import difflib

ripperdoc/tools/file_read_tool.py

@@ -3,6 +3,7 @@
 Allows the AI to read file contents.
 """
 
+import itertools
 import os
 from pathlib import Path
 from typing import AsyncGenerator, List, Optional, Tuple
@@ -24,7 +25,7 @@ from ripperdoc.utils.path_ignore import check_path_for_tool
 logger = get_logger()
 
 
-def detect_file_encoding(file_path: str) -> Tuple[Optional[str], float]:
+def detect_file_encoding(file_path: str, max_bytes: Optional[int] = None) -> Tuple[Optional[str], float]:
     """Detect file encoding using charset-normalizer.
 
     Returns:
@@ -32,7 +33,7 @@ def detect_file_encoding(file_path: str) -> Tuple[Optional[str], float]:
     """
     try:
         with open(file_path, "rb") as f:
-            raw_data = f.read()
+            raw_data = f.read() if max_bytes is None else f.read(max_bytes)
         results = from_bytes(raw_data)
 
         if not results:
@@ -115,6 +116,76 @@ def read_file_with_encoding(file_path: str) -> Tuple[Optional[List[str]], str, O
     return None, "", error_msg
 
 
+def read_file_slice_with_encoding(
+    file_path: str, offset: int, limit: Optional[int], sample_bytes: int = 65536
+) -> Tuple[Optional[List[str]], str, Optional[str]]:
+    """Read a slice of a file with encoding detection.
+
+    Returns:
+        Tuple of (lines, encoding_used, error_message).
+        If successful: (lines, encoding, None)
+        If failed: (None, "", error_message)
+    """
+
+    def _read_slice(encoding: str) -> List[str]:
+        start = max(offset, 0)
+        if limit is None:
+            end = None
+        elif limit <= 0:
+            return []
+        else:
+            end = start + limit
+        with open(file_path, "r", encoding=encoding, errors="strict") as f:
+            return list(itertools.islice(f, start, end))
+
+    # First, try UTF-8 (most common)
+    try:
+        lines = _read_slice("utf-8")
+        return lines, "utf-8", None
+    except UnicodeDecodeError:
+        pass
+
+    # UTF-8 failed, use charset-normalizer to detect encoding (sampled)
+    detected_encoding, confidence = detect_file_encoding(file_path, max_bytes=sample_bytes)
+
+    if detected_encoding:
+        try:
+            lines = _read_slice(detected_encoding)
+            logger.info(
+                "File %s decoded using detected encoding %s",
+                file_path,
+                detected_encoding,
+            )
+            return lines, detected_encoding, None
+        except (UnicodeDecodeError, LookupError) as e:
+            logger.warning(
+                "Failed to read %s with detected encoding %s: %s",
+                file_path,
+                detected_encoding,
+                e,
+            )
+
+    # Detection failed - try latin-1 as last resort (can decode any byte sequence)
+    try:
+        lines = _read_slice("latin-1")
+        logger.warning(
+            "File %s: encoding detection failed, using latin-1 fallback",
+            file_path,
+        )
+        return lines, "latin-1", None
+    except (UnicodeDecodeError, LookupError):
+        pass
+
+    # All attempts failed - return error
+    error_msg = (
+        f"Unable to determine file encoding. "
+        f"Detected: {detected_encoding or 'unknown'} (confidence: {confidence * 100:.0f}%). "
+        f"Tried fallback encodings: utf-8, latin-1. "
+        f"Please convert the file to UTF-8."
+    )
+    return None, "", error_msg
+
+
 # Maximum file size to read (default 256KB)
 # Can be overridden via env var in bytes
 MAX_FILE_SIZE_BYTES = int(os.getenv("RIPPERDOC_MAX_READ_FILE_SIZE_BYTES", "262144"))  # 256KB
@@ -243,7 +314,9 @@ and limit to read only a portion of the file."""
         try:
             # Check file size before reading to prevent memory exhaustion
             file_size = os.path.getsize(input_data.file_path)
-            if file_size > MAX_FILE_SIZE_BYTES:
+            offset = max(input_data.offset or 0, 0)
+            limit = input_data.limit
+            if file_size > MAX_FILE_SIZE_BYTES and limit is None:
                 size_kb = file_size / 1024
                 limit_kb = MAX_FILE_SIZE_BYTES / 1024
                 error_output = FileReadToolOutput(
@@ -259,8 +332,16 @@ and limit to read only a portion of the file."""
                 )
                 return
 
-            # Detect and read file with proper encoding
-            lines, used_encoding, encoding_error = read_file_with_encoding(input_data.file_path)
+            if limit is None:
+                # Detect and read full file with proper encoding
+                lines, used_encoding, encoding_error = read_file_with_encoding(
+                    input_data.file_path
+                )
+            else:
+                # Read only the requested slice (avoids loading huge files)
+                lines, used_encoding, encoding_error = read_file_slice_with_encoding(
+                    input_data.file_path, offset=offset, limit=limit
+                )
 
             if lines is None:
                 # Encoding detection failed - return warning to LLM
@@ -277,29 +358,24 @@ and limit to read only a portion of the file."""
                 )
                 return
 
-            offset = input_data.offset or 0
-            limit = input_data.limit
-            total_lines = len(lines)
-
             # Check line count if no limit is specified (to prevent context overflow)
-            if limit is None and total_lines > MAX_READ_LINES:
-                error_output = FileReadToolOutput(
-                    content=f"File too large: {total_lines} lines exceeds limit of {MAX_READ_LINES} lines. Use offset and limit parameters to read portions.",
-                    file_path=input_data.file_path,
-                    line_count=total_lines,
-                    offset=0,
-                    limit=None,
-                )
-                yield ToolResult(
-                    data=error_output,
-                    result_for_assistant=f"Error: File {input_data.file_path} has {total_lines} lines, exceeding the limit of {MAX_READ_LINES} lines when reading without limit parameter. Use offset and limit to read portions, e.g., Read(file_path='{input_data.file_path}', offset=0, limit=500).",
-                )
-                return
-
-            # Apply offset and limit
             if limit is not None:
-                selected_lines = lines[offset : offset + limit]
+                selected_lines = lines
             else:
+                total_lines = len(lines)
+                if total_lines > MAX_READ_LINES:
+                    error_output = FileReadToolOutput(
+                        content=f"File too large: {total_lines} lines exceeds limit of {MAX_READ_LINES} lines. Use offset and limit parameters to read portions.",
+                        file_path=input_data.file_path,
+                        line_count=total_lines,
+                        offset=0,
+                        limit=None,
+                    )
+                    yield ToolResult(
+                        data=error_output,
+                        result_for_assistant=f"Error: File {input_data.file_path} has {total_lines} lines, exceeding the limit of {MAX_READ_LINES} lines when reading without limit parameter. Use offset and limit to read portions, e.g., Read(file_path='{input_data.file_path}', offset=0, limit=500).",
+                    )
+                    return
                 selected_lines = lines[offset:]
 
             content = "".join(selected_lines)