tunacode-cli 0.1.21__py3-none-any.whl

tunacode/tools/utils/ripgrep.py

@@ -0,0 +1,311 @@
+"""Ripgrep binary management and execution utilities."""
+
+import functools
+import os
+import platform
+import shutil
+import subprocess
+from pathlib import Path
+
+
+@functools.lru_cache(maxsize=1)
+def get_platform_identifier() -> tuple[str, str]:
+    """Get the current platform identifier.
+
+    Returns:
+        Tuple of (platform_key, system_name)
+    """
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+
+    if system == "linux":
+        if machine in ["x86_64", "amd64"]:
+            return "x64-linux", system
+        elif machine in ["aarch64", "arm64"]:
+            return "arm64-linux", system
+    elif system == "darwin":  # noqa: SIM102
+        if machine in ["x86_64", "amd64"]:
+            return "x64-darwin", system
+        elif machine in ["arm64", "aarch64"]:
+            return "arm64-darwin", system
+    elif system == "windows":  # noqa: SIM102
+        if machine in ["x86_64", "amd64"]:
+            return "x64-win32", system
+
+    raise ValueError(f"Unsupported platform: {system} {machine}")
+
+
+@functools.lru_cache(maxsize=1)
+def get_ripgrep_binary_path() -> Path | None:
+    """Resolve the path to the ripgrep binary.
+
+    Resolution order:
+    1. Environment variable override (TUNACODE_RIPGREP_PATH)
+    2. System ripgrep (if newer or equal version)
+    3. Bundled ripgrep binary
+    4. None (fallback to Python-based search)
+
+    Returns:
+        Path to ripgrep binary or None if not available
+    """
+    # Check for environment variable override
+    env_path = os.environ.get("TUNACODE_RIPGREP_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file():
+            return path
+
+    # Check for system ripgrep
+    system_rg = shutil.which("rg")
+    if system_rg:
+        system_rg_path = Path(system_rg)
+        if _check_ripgrep_version(system_rg_path):
+            return system_rg_path
+
+    # Check for bundled ripgrep
+    try:
+        platform_key, _ = get_platform_identifier()
+        binary_name = "rg.exe" if platform_key == "x64-win32" else "rg"
+
+        # Look for vendor directory relative to this file
+        vendor_dir = (
+            Path(__file__).parent.parent.parent.parent / "vendor" / "ripgrep" / platform_key
+        )
+        bundled_path = vendor_dir / binary_name
+
+        if bundled_path.exists():
+            return bundled_path
+    except Exception:
+        pass
+
+    return None
+
+
+def _check_ripgrep_version(rg_path: Path, min_version: str = "13.0.0") -> bool:
+    """Check if ripgrep version meets minimum requirement.
+
+    Args:
+        rg_path: Path to ripgrep binary
+        min_version: Minimum required version
+
+    Returns:
+        True if version is sufficient, False otherwise
+    """
+    try:
+        result = subprocess.run(
+            [str(rg_path), "--version"],
+            capture_output=True,
+            text=True,
+            timeout=1,
+        )
+        if result.returncode == 0:
+            # Parse version from output like "ripgrep 14.1.1"
+            version_line = result.stdout.split("\n")[0]
+            version = version_line.split()[-1]
+
+            # Simple version comparison (works for x.y.z format)
+            current = tuple(map(int, version.split(".")))
+            required = tuple(map(int, min_version.split(".")))
+
+            return current >= required
+    except Exception:
+        pass
+
+    return False
+
+
+class RipgrepExecutor:
+    """Wrapper for executing ripgrep commands with error handling."""
+
+    def __init__(self, binary_path: Path | None = None):
+        """Initialize the executor.
+
+        Args:
+            binary_path: Optional path to ripgrep binary
+        """
+        self.binary_path = binary_path or get_ripgrep_binary_path()
+        self._use_python_fallback = self.binary_path is None
+
+    def search(
+        self,
+        pattern: str,
+        path: str = ".",
+        *,
+        timeout: int = 10,
+        max_matches: int | None = None,
+        file_pattern: str | None = None,
+        case_insensitive: bool = False,
+        multiline: bool = False,
+        context_before: int = 0,
+        context_after: int = 0,
+        **kwargs,
+    ) -> list[str]:
+        """Execute a ripgrep search.
+
+        Args:
+            pattern: Search pattern (regex)
+            path: Directory or file to search
+            timeout: Maximum execution time in seconds
+            max_matches: Maximum number of matches to return
+            file_pattern: Glob pattern for files to include
+            case_insensitive: Case-insensitive search
+            multiline: Enable multiline mode
+            context_before: Lines of context before match
+            context_after: Lines of context after match
+            **kwargs: Additional ripgrep arguments
+
+        Returns:
+            List of matching lines or file paths
+        """
+        if self._use_python_fallback:
+            return self._python_fallback_search(
+                pattern, path, file_pattern=file_pattern, case_insensitive=case_insensitive
+            )
+
+        try:
+            cmd = [str(self.binary_path)]
+
+            # Add flags
+            if case_insensitive:
+                cmd.append("-i")
+            if multiline:
+                cmd.extend(["-U", "--multiline-dotall"])
+            if context_before > 0:
+                cmd.extend(["-B", str(context_before)])
+            if context_after > 0:
+                cmd.extend(["-A", str(context_after)])
+            if max_matches:
+                cmd.extend(["-m", str(max_matches)])
+            if file_pattern:
+                cmd.extend(["-g", file_pattern])
+
+            # Add pattern and path
+            cmd.extend([pattern, path])
+
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+
+            if result.returncode in [0, 1]:  # 0 = matches found, 1 = no matches
+                return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+            else:
+                return []
+
+        except subprocess.TimeoutExpired:
+            return []
+        except Exception:
+            return self._python_fallback_search(pattern, path, file_pattern=file_pattern)
+
+    def list_files(self, pattern: str, directory: str = ".") -> list[str]:
+        """List files matching a glob pattern using ripgrep.
+
+        Args:
+            pattern: Glob pattern for files
+            directory: Directory to search
+
+        Returns:
+            List of file paths
+        """
+        if self._use_python_fallback:
+            return self._python_fallback_list_files(pattern, directory)
+
+        try:
+            result = subprocess.run(
+                [str(self.binary_path), "--files", "-g", pattern, directory],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+        except Exception:
+            return self._python_fallback_list_files(pattern, directory)
+
+    def _python_fallback_search(
+        self,
+        pattern: str,
+        path: str,
+        file_pattern: str | None = None,
+        case_insensitive: bool = False,
+    ) -> list[str]:
+        """Python-based fallback search implementation."""
+        import re
+        from pathlib import Path
+
+        results = []
+        path_obj = Path(path)
+
+        # Compile regex pattern
+        flags = re.IGNORECASE if case_insensitive else 0
+        try:
+            regex = re.compile(pattern, flags)
+        except re.error:
+            return []
+
+        # Search files
+        if path_obj.is_file():
+            files = [path_obj]
+        else:
+            glob_pattern = file_pattern or "**/*"
+            files = list(path_obj.glob(glob_pattern))
+
+        for file_path in files:
+            if not file_path.is_file():
+                continue
+
+            try:
+                with file_path.open("r", encoding="utf-8", errors="ignore") as f:
+                    for line_num, line in enumerate(f, 1):
+                        if regex.search(line):
+                            results.append(f"{file_path}:{line_num}:{line.strip()}")
+            except Exception:  # nosec B112 - continue on file read errors is appropriate
+                continue
+
+        return results
+
+    def _python_fallback_list_files(self, pattern: str, directory: str) -> list[str]:
+        """Python-based fallback for listing files."""
+        from pathlib import Path
+
+        try:
+            base_path = Path(directory)
+            return [str(p) for p in base_path.glob(pattern) if p.is_file()]
+        except Exception:
+            return []
+
+
+# Performance metrics collection
+class RipgrepMetrics:
+    """Collect performance metrics for ripgrep operations."""
+
+    def __init__(self):
+        self.search_count = 0
+        self.total_search_time = 0.0
+        self.fallback_count = 0
+
+    def record_search(self, duration: float, used_fallback: bool = False):
+        """Record a search operation."""
+        self.search_count += 1
+        self.total_search_time += duration
+        if used_fallback:
+            self.fallback_count += 1
+
+    @property
+    def average_search_time(self) -> float:
+        """Get average search time."""
+        if self.search_count == 0:
+            return 0.0
+        return self.total_search_time / self.search_count
+
+    @property
+    def fallback_rate(self) -> float:
+        """Get fallback usage rate."""
+        if self.search_count == 0:
+            return 0.0
+        return self.fallback_count / self.search_count
+
+
+# Global metrics instance
+metrics = RipgrepMetrics()

tunacode/tools/utils/text_match.py

@@ -0,0 +1,352 @@
+"""
+Module: tunacode.tools.edit_replacers
+
+Fuzzy string replacement strategies for the update_file tool.
+Inspired by:
+- https://github.com/cline/cline/blob/main/evals/diff-edits/diff-apply/diff-06-23-25.ts
+- https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/utils/editCorrector.ts
+- https://github.com/sst/opencode/blob/dev/packages/opencode/src/tool/edit.ts
+
+Each replacer is a generator that yields potential matches found in content.
+Replacers are tried in order from strict to fuzzy until one succeeds.
+"""
+
+from collections.abc import Callable, Generator
+
+try:
+    from Levenshtein import distance as _levenshtein_c
+
+    _USE_C_LEVENSHTEIN = True
+except ImportError:
+    _USE_C_LEVENSHTEIN = False
+
+# Type alias for replacer functions
+Replacer = Callable[[str, str], Generator[str, None, None]]
+
+# Similarity thresholds for block anchor fallback matching
+SINGLE_CANDIDATE_SIMILARITY_THRESHOLD = 0.0
+MULTIPLE_CANDIDATES_SIMILARITY_THRESHOLD = 0.3
+
+
+def levenshtein(a: str, b: str) -> int:
+    """Levenshtein edit distance between two strings."""
+    if _USE_C_LEVENSHTEIN:
+        return _levenshtein_c(a, b)
+
+    if not a or not b:
+        return max(len(a), len(b))
+
+    matrix = [[0] * (len(b) + 1) for _ in range(len(a) + 1)]
+    for i in range(len(a) + 1):
+        matrix[i][0] = i
+    for j in range(len(b) + 1):
+        matrix[0][j] = j
+
+    for i in range(1, len(a) + 1):
+        for j in range(1, len(b) + 1):
+            cost = 0 if a[i - 1] == b[j - 1] else 1
+            matrix[i][j] = min(
+                matrix[i - 1][j] + 1,  # deletion
+                matrix[i][j - 1] + 1,  # insertion
+                matrix[i - 1][j - 1] + cost,  # substitution
+            )
+    return matrix[len(a)][len(b)]
+
+
+def simple_replacer(content: str, find: str) -> Generator[str, None, None]:
+    """Exact match replacer - yields find string if present in content.
+
+    This is the strictest matcher and preserves backwards compatibility.
+    """
+    if find in content:
+        yield find
+
+
+def line_trimmed_replacer(content: str, find: str) -> Generator[str, None, None]:
+    """Match lines ignoring leading/trailing whitespace per line.
+
+    Compares trimmed versions of each line but yields the original
+    content with its actual whitespace preserved.
+    """
+    # Use splitlines(keepends=True) to handle both \n and \r\n correctly
+    original_lines = content.splitlines(keepends=True)
+    search_lines = find.split("\n")
+
+    # Remove trailing empty line if present (common from copy-paste)
+    if search_lines and search_lines[-1] == "":
+        search_lines.pop()
+
+    if not search_lines:
+        return
+
+    for i in range(len(original_lines) - len(search_lines) + 1):
+        matches = True
+
+        for j in range(len(search_lines)):
+            original_trimmed = original_lines[i + j].strip()
+            search_trimmed = search_lines[j].strip()
+
+            if original_trimmed != search_trimmed:
+                matches = False
+                break
+
+        if matches:
+            # Join the matched lines and strip trailing line ending
+            matched_block = "".join(original_lines[i : i + len(search_lines)])
+            yield matched_block.rstrip("\r\n")
+
+
+def indentation_flexible_replacer(content: str, find: str) -> Generator[str, None, None]:
+    """Match blocks after normalizing indentation.
+
+    Strips the minimum common indentation from both content blocks
+    and find string, then compares. Useful when the LLM gets the
+    indentation level wrong but the code structure is correct.
+    """
+
+    def remove_indentation(text: str) -> str:
+        """Remove common leading indentation from all lines."""
+        lines = text.split("\n")
+        non_empty_lines = [line for line in lines if line.strip()]
+
+        if not non_empty_lines:
+            return text
+
+        # Find minimum indentation
+        min_indent = float("inf")
+        for line in non_empty_lines:
+            stripped = line.lstrip()
+            if stripped:
+                indent = len(line) - len(stripped)
+                min_indent = min(min_indent, indent)
+
+        if min_indent == float("inf") or min_indent == 0:
+            return text
+
+        # Remove the common indentation
+        result_lines = []
+        for line in lines:
+            if line.strip():
+                result_lines.append(line[int(min_indent) :])
+            else:
+                result_lines.append(line)
+
+        return "\n".join(result_lines)
+
+    content_lines = content.split("\n")
+    find_lines = find.split("\n")
+
+    # Remove trailing empty line if present (MUST happen before normalization)
+    if find_lines and find_lines[-1] == "":
+        find_lines.pop()
+
+    if not find_lines:
+        return
+
+    # Normalize find AFTER trimming trailing empty line
+    normalized_find = remove_indentation("\n".join(find_lines))
+
+    # Pre-compute first line stripped for fail-fast check
+    first_find_stripped = find_lines[0].strip()
+
+    for i in range(len(content_lines) - len(find_lines) + 1):
+        # Fail-fast: skip if first line doesn't match (ignoring indentation)
+        if content_lines[i].strip() != first_find_stripped:
+            continue
+
+        block = "\n".join(content_lines[i : i + len(find_lines)])
+        if remove_indentation(block) == normalized_find:
+            yield block
+
+
+def block_anchor_replacer(content: str, find: str) -> Generator[str, None, None]:
+    """Match using first/last lines as anchors, fuzzy-match middle.
+
+    This is the fuzziest matcher. It:
+    1. Finds blocks where first and last lines match exactly (trimmed)
+    2. Uses Levenshtein distance to score middle line similarity
+    3. Returns best match if similarity exceeds threshold
+    """
+    # Use splitlines(keepends=True) to handle both \n and \r\n correctly
+    original_lines = content.splitlines(keepends=True)
+    search_lines = find.split("\n")
+
+    # Need at least 3 lines for anchor matching to make sense
+    if len(search_lines) < 3:
+        return
+
+    # Remove trailing empty line if present
+    if search_lines[-1] == "":
+        search_lines.pop()
+
+    if len(search_lines) < 3:
+        return
+
+    first_line_search = search_lines[0].strip()
+    last_line_search = search_lines[-1].strip()
+    search_block_size = len(search_lines)
+
+    # Build indices in O(n) instead of O(n^2) nested loop
+    first_indices = [
+        i for i, line in enumerate(original_lines) if line.strip() == first_line_search
+    ]
+    last_indices = [j for j, line in enumerate(original_lines) if line.strip() == last_line_search]
+
+    if not first_indices or not last_indices:
+        return
+
+    # Find valid pairs in O(k^2) where k << n typically
+    candidates: list[tuple[int, int]] = []
+    for i in first_indices:
+        for j in last_indices:
+            if j > i + 1:  # Need at least one line between anchors
+                candidates.append((i, j))
+                break  # Only first matching last anchor after this first
+
+    if not candidates:
+        return
+
+    def yield_block(start: int, end: int) -> str:
+        """Join lines from start to end (inclusive) and strip trailing line ending."""
+        return "".join(original_lines[start : end + 1]).rstrip("\r\n")
+
+    # Handle single candidate (use relaxed threshold)
+    if len(candidates) == 1:
+        start_line, end_line = candidates[0]
+        actual_block_size = end_line - start_line + 1
+
+        similarity = 0.0
+        lines_to_check = min(search_block_size - 2, actual_block_size - 2)
+
+        if lines_to_check > 0:
+            for j in range(1, min(search_block_size - 1, actual_block_size - 1)):
+                original_line = original_lines[start_line + j].strip()
+                search_line = search_lines[j].strip()
+                max_len = max(len(original_line), len(search_line))
+
+                if max_len == 0:
+                    continue
+
+                distance = levenshtein(original_line, search_line)
+                similarity += (1 - distance / max_len) / lines_to_check
+
+                if similarity >= SINGLE_CANDIDATE_SIMILARITY_THRESHOLD:
+                    break
+        else:
+            # No middle lines to compare, accept based on anchors
+            similarity = 1.0
+
+        if similarity >= SINGLE_CANDIDATE_SIMILARITY_THRESHOLD:
+            yield yield_block(start_line, end_line)
+        return
+
+    # Multiple candidates - find best match
+    best_match: tuple[int, int] | None = None
+    max_similarity = -1.0
+
+    for start_line, end_line in candidates:
+        actual_block_size = end_line - start_line + 1
+
+        similarity = 0.0
+        lines_to_check = min(search_block_size - 2, actual_block_size - 2)
+
+        if lines_to_check > 0:
+            for j in range(1, min(search_block_size - 1, actual_block_size - 1)):
+                original_line = original_lines[start_line + j].strip()
+                search_line = search_lines[j].strip()
+                max_len = max(len(original_line), len(search_line))
+
+                if max_len == 0:
+                    continue
+
+                distance = levenshtein(original_line, search_line)
+                similarity += 1 - distance / max_len
+
+            similarity /= lines_to_check  # Average similarity
+        else:
+            similarity = 1.0
+
+        if similarity > max_similarity:
+            max_similarity = similarity
+            best_match = (start_line, end_line)
+
+    # Check threshold for multiple candidates
+    if max_similarity >= MULTIPLE_CANDIDATES_SIMILARITY_THRESHOLD and best_match:
+        start_line, end_line = best_match
+        yield yield_block(start_line, end_line)
+
+
+# Ordered list of replacers from strict to fuzzy
+REPLACERS: list[Replacer] = [
+    simple_replacer,
+    line_trimmed_replacer,
+    indentation_flexible_replacer,
+    block_anchor_replacer,
+]
+
+
+def replace(content: str, old_string: str, new_string: str, replace_all: bool = False) -> str:
+    """Replace old_string with new_string using fuzzy matching.
+
+    Tries each replacer in order until one succeeds. Replacers are ordered
+    from strict (exact match) to fuzzy (anchor-based with Levenshtein).
+
+    Args:
+        content: The file content to modify
+        old_string: The text to find and replace
+        new_string: The replacement text
+        replace_all: If True, replace all occurrences; if False, require unique match
+
+    Returns:
+        Modified content with replacement applied
+
+    Raises:
+        ValueError: If old_string equals new_string
+        ValueError: If no match found after trying all strategies
+        ValueError: If multiple matches found and replace_all is False
+    """
+    if old_string == "":
+        raise ValueError("old_string cannot be empty; handle file overwrite separately")
+
+    if old_string == new_string:
+        raise ValueError("old_string and new_string must be different")
+
+    found_multiple = False
+    found_fuzzy_match_for_replace_all = False
+
+    for replacer_idx, replacer in enumerate(REPLACERS):
+        is_exact_match = replacer_idx == 0  # simple_replacer is exact
+
+        for search in replacer(content, old_string):
+            index = content.find(search)
+            if index == -1:
+                continue
+
+            if replace_all:
+                if not is_exact_match:
+                    # Fuzzy match with replace_all is risky - track but don't use
+                    found_fuzzy_match_for_replace_all = True
+                    continue
+                return content.replace(search, new_string)
+
+            # Check for uniqueness - only replace if single occurrence
+            last_index = content.rfind(search)
+            if index != last_index:
+                found_multiple = True
+                continue  # Try next replacer for potentially more specific match
+
+            return content[:index] + new_string + content[index + len(search) :]
+
+    if found_fuzzy_match_for_replace_all:
+        raise ValueError(
+            "replace_all=True only allowed with exact matches. "
+            "Fuzzy matching found a match but replace_all is risky for non-exact matches."
+        )
+
+    if found_multiple:
+        raise ValueError(
+            "Found multiple matches for old_string. "
+            "Provide more surrounding lines in old_string to identify the correct match."
+        )
+
+    raise ValueError("old_string not found in content (tried all fuzzy matching strategies)")