tunacode-cli 0.0.55__py3-none-any.whl → 0.0.57__py3-none-any.whl

tunacode/utils/json_utils.py

@@ -0,0 +1,206 @@
+"""
+Module: tunacode.utils.json_utils
+
+JSON parsing utilities with enhanced error handling and concatenated object support.
+"""
+
+import json
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from tunacode.constants import READ_ONLY_TOOLS
+
+logger = logging.getLogger(__name__)
+
+
+class ConcatenatedJSONError(Exception):
+    """Raised when concatenated JSON objects are detected but cannot be safely handled."""
+
+    def __init__(self, message: str, objects_found: int, tool_name: Optional[str] = None):
+        self.message = message
+        self.objects_found = objects_found
+        self.tool_name = tool_name
+        super().__init__(message)
+
+
+def split_concatenated_json(json_string: str, strict_mode: bool = True) -> List[Dict[str, Any]]:
+    """
+    Split concatenated JSON objects like {"a": 1}{"b": 2} into separate objects.
+
+    Args:
+        json_string: String containing potentially concatenated JSON objects
+        strict_mode: If True, only returns valid JSON objects. If False, attempts
+                    to recover partial objects.
+
+    Returns:
+        List of parsed JSON objects
+
+    Raises:
+        json.JSONDecodeError: If no valid JSON objects can be extracted
+        ConcatenatedJSONError: If multiple objects found but not safe to process
+    """
+    objects = []
+    brace_count = 0
+    start_pos = 0
+    in_string = False
+    escape_next = False
+
+    for i, char in enumerate(json_string):
+        if escape_next:
+            escape_next = False
+            continue
+
+        if char == "\\":
+            escape_next = True
+            continue
+
+        if char == '"' and not escape_next:
+            in_string = not in_string
+            continue
+
+        if in_string:
+            continue
+
+        if char == "{":
+            if brace_count == 0:
+                start_pos = i
+            brace_count += 1
+        elif char == "}":
+            brace_count -= 1
+            if brace_count == 0:
+                potential_json = json_string[start_pos : i + 1].strip()
+                try:
+                    parsed = json.loads(potential_json)
+                    if isinstance(parsed, dict):
+                        objects.append(parsed)
+                    else:
+                        logger.warning(f"Non-dict JSON object ignored: {type(parsed)}")
+                except json.JSONDecodeError as e:
+                    if strict_mode:
+                        logger.debug(f"Invalid JSON fragment skipped: {potential_json[:100]}...")
+                    else:
+                        logger.warning(f"JSON parse error: {e}")
+                    continue
+
+    if not objects:
+        raise json.JSONDecodeError("No valid JSON objects found", json_string, 0)
+
+    return objects
+
+
+def validate_tool_args_safety(
+    objects: List[Dict[str, Any]], tool_name: Optional[str] = None
+) -> bool:
+    """
+    Validate whether it's safe to execute multiple JSON objects for a given tool.
+
+    Args:
+        objects: List of JSON objects to validate
+        tool_name: Name of the tool (if known)
+
+    Returns:
+        bool: True if safe to execute, False otherwise
+
+    Raises:
+        ConcatenatedJSONError: If multiple objects detected for unsafe tool
+    """
+    if len(objects) <= 1:
+        return True
+
+    # Check if tool is read-only (safer to execute multiple times)
+    if tool_name and tool_name in READ_ONLY_TOOLS:
+        logger.info(f"Multiple JSON objects for read-only tool {tool_name} - allowing execution")
+        return True
+
+    # For write/execute tools, multiple objects are potentially dangerous
+    if tool_name:
+        logger.warning(
+            f"Multiple JSON objects detected for tool {tool_name} "
+            f"({len(objects)} objects). This may indicate a model error."
+        )
+        raise ConcatenatedJSONError(
+            f"Multiple JSON objects not safe for tool {tool_name}",
+            objects_found=len(objects),
+            tool_name=tool_name,
+        )
+    else:
+        logger.warning(f"Multiple JSON objects detected ({len(objects)}) with unknown tool")
+        return False
+
+
+def safe_json_parse(
+    json_string: str, tool_name: Optional[str] = None, allow_concatenated: bool = False
+) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+    """
+    Safely parse JSON with optional concatenated object support.
+
+    Args:
+        json_string: JSON string to parse
+        tool_name: Name of the tool (for safety validation)
+        allow_concatenated: Whether to attempt splitting concatenated objects
+
+    Returns:
+        Single dict if one object, or list of dicts if multiple objects
+
+    Raises:
+        json.JSONDecodeError: If parsing fails
+        ConcatenatedJSONError: If concatenated objects are unsafe
+    """
+    try:
+        # First, try normal JSON parsing
+        result = json.loads(json_string)
+        if isinstance(result, dict):
+            return result
+        else:
+            raise json.JSONDecodeError(f"Expected dict, got {type(result)}", json_string, 0)
+
+    except json.JSONDecodeError as e:
+        if not allow_concatenated or "Extra data" not in str(e):
+            raise
+
+        logger.info("Attempting to split concatenated JSON objects")
+
+        # Try to split concatenated objects
+        objects = split_concatenated_json(json_string)
+
+        # Validate safety
+        if validate_tool_args_safety(objects, tool_name):
+            if len(objects) == 1:
+                return objects[0]
+            else:
+                return objects
+        else:
+            # Not safe - return first object with warning
+            logger.warning(f"Using first of {len(objects)} JSON objects only")
+            return objects[0]
+
+
+def merge_json_objects(objects: List[Dict[str, Any]], strategy: str = "first") -> Dict[str, Any]:
+    """
+    Merge multiple JSON objects using different strategies.
+
+    Args:
+        objects: List of JSON objects to merge
+        strategy: Merge strategy ("first", "last", "combine")
+
+    Returns:
+        Single merged JSON object
+    """
+    if not objects:
+        return {}
+
+    if len(objects) == 1:
+        return objects[0]
+
+    if strategy == "first":
+        return objects[0]
+    elif strategy == "last":
+        return objects[-1]
+    elif strategy == "combine":
+        # Combine all objects, later values override earlier ones
+        result = {}
+        for obj in objects:
+            result.update(obj)
+        return result
+    else:
+        raise ValueError(f"Unknown merge strategy: {strategy}")

tunacode/utils/message_utils.py

@@ -9,11 +9,21 @@ def get_message_content(message: Any) -> str:
         return message
     if isinstance(message, dict):
         if "content" in message:
-            return message["content"]
+            content = message["content"]
+            # Handle nested content structures
+            if isinstance(content, list):
+                return " ".join(get_message_content(item) for item in content)
+            return str(content)
         if "thought" in message:
-            return message["thought"]
+            return str(message["thought"])
     if hasattr(message, "content"):
-        return message.content
+        content = message.content
+        if isinstance(content, list):
+            return " ".join(get_message_content(item) for item in content)
+        return str(content)
     if hasattr(message, "parts"):
-        return " ".join(get_message_content(part) for part in message.parts)
+        parts = message.parts
+        if isinstance(parts, list):
+            return " ".join(get_message_content(part) for part in parts)
+        return str(parts)
     return ""

tunacode/utils/ripgrep.py

@@ -1,17 +1,340 @@
+"""Ripgrep binary management and execution utilities."""
+
+import functools
+import logging
+import os
+import platform
+import shutil
 import subprocess
-from typing import List
+from pathlib import Path
+from typing import List, Optional, Tuple
 
+logger = logging.getLogger(__name__)
 
-def ripgrep(pattern: str, directory: str = ".") -> List[str]:
-    """Return a list of file paths matching a pattern using ripgrep."""
+
+@functools.lru_cache(maxsize=1)
+def get_platform_identifier() -> Tuple[str, str]:
+    """Get the current platform identifier.
+
+    Returns:
+        Tuple of (platform_key, system_name)
+    """
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+
+    if system == "linux":
+        if machine in ["x86_64", "amd64"]:
+            return "x64-linux", system
+        elif machine in ["aarch64", "arm64"]:
+            return "arm64-linux", system
+    elif system == "darwin":
+        if machine in ["x86_64", "amd64"]:
+            return "x64-darwin", system
+        elif machine in ["arm64", "aarch64"]:
+            return "arm64-darwin", system
+    elif system == "windows":
+        if machine in ["x86_64", "amd64"]:
+            return "x64-win32", system
+
+    raise ValueError(f"Unsupported platform: {system} {machine}")
+
+
+@functools.lru_cache(maxsize=1)
+def get_ripgrep_binary_path() -> Optional[Path]:
+    """Resolve the path to the ripgrep binary.
+
+    Resolution order:
+    1. Environment variable override (TUNACODE_RIPGREP_PATH)
+    2. System ripgrep (if newer or equal version)
+    3. Bundled ripgrep binary
+    4. None (fallback to Python-based search)
+
+    Returns:
+        Path to ripgrep binary or None if not available
+    """
+    # Check for environment variable override
+    env_path = os.environ.get("TUNACODE_RIPGREP_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.exists() and path.is_file():
+            logger.debug(f"Using ripgrep from environment variable: {path}")
+            return path
+        else:
+            logger.warning(f"Invalid TUNACODE_RIPGREP_PATH: {env_path}")
+
+    # Check for system ripgrep
+    system_rg = shutil.which("rg")
+    if system_rg:
+        system_rg_path = Path(system_rg)
+        if _check_ripgrep_version(system_rg_path):
+            logger.debug(f"Using system ripgrep: {system_rg_path}")
+            return system_rg_path
+
+    # Check for bundled ripgrep
+    try:
+        platform_key, _ = get_platform_identifier()
+        binary_name = "rg.exe" if platform_key == "x64-win32" else "rg"
+
+        # Look for vendor directory relative to this file
+        vendor_dir = (
+            Path(__file__).parent.parent.parent.parent / "vendor" / "ripgrep" / platform_key
+        )
+        bundled_path = vendor_dir / binary_name
+
+        if bundled_path.exists():
+            logger.debug(f"Using bundled ripgrep: {bundled_path}")
+            return bundled_path
+    except Exception as e:
+        logger.debug(f"Could not find bundled ripgrep: {e}")
+
+    logger.debug("No ripgrep binary found, will use Python fallback")
+    return None
+
+
+def _check_ripgrep_version(rg_path: Path, min_version: str = "13.0.0") -> bool:
+    """Check if ripgrep version meets minimum requirement.
+
+    Args:
+        rg_path: Path to ripgrep binary
+        min_version: Minimum required version
+
+    Returns:
+        True if version is sufficient, False otherwise
+    """
     try:
         result = subprocess.run(
-            ["rg", "--files", "-g", pattern, directory],
+            [str(rg_path), "--version"],
             capture_output=True,
             text=True,
-            check=True,
-            timeout=5,
+            timeout=1,
         )
-        return [line.strip() for line in result.stdout.splitlines() if line.strip()]
-    except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
-        return []
+        if result.returncode == 0:
+            # Parse version from output like "ripgrep 14.1.1"
+            version_line = result.stdout.split("\n")[0]
+            version = version_line.split()[-1]
+
+            # Simple version comparison (works for x.y.z format)
+            current = tuple(map(int, version.split(".")))
+            required = tuple(map(int, min_version.split(".")))
+
+            return current >= required
+    except Exception as e:
+        logger.debug(f"Could not check ripgrep version: {e}")
+
+    return False
+
+
+class RipgrepExecutor:
+    """Wrapper for executing ripgrep commands with error handling."""
+
+    def __init__(self, binary_path: Optional[Path] = None):
+        """Initialize the executor.
+
+        Args:
+            binary_path: Optional path to ripgrep binary
+        """
+        self.binary_path = binary_path or get_ripgrep_binary_path()
+        self._use_python_fallback = self.binary_path is None
+
+        if self._use_python_fallback:
+            logger.info("Ripgrep binary not available, using Python fallback")
+
+    def search(
+        self,
+        pattern: str,
+        path: str = ".",
+        *,
+        timeout: int = 10,
+        max_matches: Optional[int] = None,
+        file_pattern: Optional[str] = None,
+        case_insensitive: bool = False,
+        multiline: bool = False,
+        context_before: int = 0,
+        context_after: int = 0,
+        **kwargs,
+    ) -> List[str]:
+        """Execute a ripgrep search.
+
+        Args:
+            pattern: Search pattern (regex)
+            path: Directory or file to search
+            timeout: Maximum execution time in seconds
+            max_matches: Maximum number of matches to return
+            file_pattern: Glob pattern for files to include
+            case_insensitive: Case-insensitive search
+            multiline: Enable multiline mode
+            context_before: Lines of context before match
+            context_after: Lines of context after match
+            **kwargs: Additional ripgrep arguments
+
+        Returns:
+            List of matching lines or file paths
+        """
+        if self._use_python_fallback:
+            return self._python_fallback_search(
+                pattern, path, file_pattern=file_pattern, case_insensitive=case_insensitive
+            )
+
+        try:
+            cmd = [str(self.binary_path)]
+
+            # Add flags
+            if case_insensitive:
+                cmd.append("-i")
+            if multiline:
+                cmd.extend(["-U", "--multiline-dotall"])
+            if context_before > 0:
+                cmd.extend(["-B", str(context_before)])
+            if context_after > 0:
+                cmd.extend(["-A", str(context_after)])
+            if max_matches:
+                cmd.extend(["-m", str(max_matches)])
+            if file_pattern:
+                cmd.extend(["-g", file_pattern])
+
+            # Add pattern and path
+            cmd.extend([pattern, path])
+
+            logger.debug(f"Executing ripgrep: {' '.join(cmd)}")
+
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+
+            if result.returncode in [0, 1]:  # 0 = matches found, 1 = no matches
+                return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+            else:
+                logger.warning(f"Ripgrep error: {result.stderr}")
+                return []
+
+        except subprocess.TimeoutExpired:
+            logger.warning(f"Ripgrep search timed out after {timeout} seconds")
+            return []
+        except Exception as e:
+            logger.error(f"Ripgrep execution failed: {e}")
+            return self._python_fallback_search(pattern, path, file_pattern=file_pattern)
+
+    def list_files(self, pattern: str, directory: str = ".") -> List[str]:
+        """List files matching a glob pattern using ripgrep.
+
+        Args:
+            pattern: Glob pattern for files
+            directory: Directory to search
+
+        Returns:
+            List of file paths
+        """
+        if self._use_python_fallback:
+            return self._python_fallback_list_files(pattern, directory)
+
+        try:
+            result = subprocess.run(
+                [str(self.binary_path), "--files", "-g", pattern, directory],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+        except Exception:
+            return self._python_fallback_list_files(pattern, directory)
+
+    def _python_fallback_search(
+        self,
+        pattern: str,
+        path: str,
+        file_pattern: Optional[str] = None,
+        case_insensitive: bool = False,
+    ) -> List[str]:
+        """Python-based fallback search implementation."""
+        import re
+        from pathlib import Path
+
+        results = []
+        path_obj = Path(path)
+
+        # Compile regex pattern
+        flags = re.IGNORECASE if case_insensitive else 0
+        try:
+            regex = re.compile(pattern, flags)
+        except re.error:
+            logger.error(f"Invalid regex pattern: {pattern}")
+            return []
+
+        # Search files
+        if path_obj.is_file():
+            files = [path_obj]
+        else:
+            glob_pattern = file_pattern or "**/*"
+            files = list(path_obj.glob(glob_pattern))
+
+        for file_path in files:
+            if not file_path.is_file():
+                continue
+
+            try:
+                with file_path.open("r", encoding="utf-8", errors="ignore") as f:
+                    for line_num, line in enumerate(f, 1):
+                        if regex.search(line):
+                            results.append(f"{file_path}:{line_num}:{line.strip()}")
+            except Exception:  # nosec B112 - continue on file read errors is appropriate
+                continue
+
+        return results
+
+    def _python_fallback_list_files(self, pattern: str, directory: str) -> List[str]:
+        """Python-based fallback for listing files."""
+        from pathlib import Path
+
+        try:
+            base_path = Path(directory)
+            return [str(p) for p in base_path.glob(pattern) if p.is_file()]
+        except Exception:
+            return []
+
+
+# Maintain backward compatibility
+def ripgrep(pattern: str, directory: str = ".") -> List[str]:
+    """Return a list of file paths matching a pattern using ripgrep.
+
+    This function maintains backward compatibility with the original implementation.
+    """
+    executor = RipgrepExecutor()
+    return executor.list_files(pattern, directory)
+
+
+# Performance metrics collection
+class RipgrepMetrics:
+    """Collect performance metrics for ripgrep operations."""
+
+    def __init__(self):
+        self.search_count = 0
+        self.total_search_time = 0.0
+        self.fallback_count = 0
+
+    def record_search(self, duration: float, used_fallback: bool = False):
+        """Record a search operation."""
+        self.search_count += 1
+        self.total_search_time += duration
+        if used_fallback:
+            self.fallback_count += 1
+
+    @property
+    def average_search_time(self) -> float:
+        """Get average search time."""
+        if self.search_count == 0:
+            return 0.0
+        return self.total_search_time / self.search_count
+
+    @property
+    def fallback_rate(self) -> float:
+        """Get fallback usage rate."""
+        if self.search_count == 0:
+            return 0.0
+        return self.fallback_count / self.search_count
+
+
+# Global metrics instance
+metrics = RipgrepMetrics()

{tunacode_cli-0.0.55.dist-info → tunacode_cli-0.0.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tunacode-cli
-Version: 0.0.55
+Version: 0.0.57
 Summary: Your agentic CLI developer.
 Author-email: larock22 <noreply@github.com>
 License: MIT
@@ -29,6 +29,7 @@ Requires-Dist: rich==14.0.0
 Requires-Dist: tiktoken>=0.5.2
 Provides-Extra: dev
 Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -39,6 +40,10 @@ Requires-Dist: vulture>=2.7; extra == "dev"
 Requires-Dist: unimport>=1.0.0; extra == "dev"
 Requires-Dist: autoflake>=2.0.0; extra == "dev"
 Requires-Dist: dead>=1.5.0; extra == "dev"
+Requires-Dist: hatch>=1.6.0; extra == "dev"
+Requires-Dist: defusedxml; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: bandit; extra == "dev"
 Dynamic: license-file
 
 # TunaCode CLI
@@ -106,7 +111,7 @@ tunacode --model "anthropic:claude-3.5-sonnet" --key "sk-ant-your-anthropic-key"
 tunacode --model "openrouter:openai/gpt-4o" --key "sk-or-your-openrouter-key"
 ```
 
-Your config is saved to `~/.config/tunacode.json` (edit directly with `nvim ~/.config/tunacode.json`)
+Your config is saved to `~/.config/tunacode.json`. This file stores your API keys, model preferences, and runtime settings like `max_iterations` (default: 40) and `context_window_size`. You can edit it directly with `nvim ~/.config/tunacode.json` or see [the complete configuration example](documentation/configuration/config-file-example.md) for all available options.
 
 ### Recommended Models
 
@@ -243,7 +248,7 @@ tunacode --model "anthropic:claude-3.5-sonnet" --key "sk-ant-your-anthropic-key"
 tunacode --model "openrouter:openai/gpt-4o" --key "sk-or-your-openrouter-key"
 ```
 
-Your config is saved to `~/.config/tunacode.json` (edit directly with `nvim ~/.config/tunacode.json`)
+Your config is saved to `~/.config/tunacode.json`. This file stores your API keys, model preferences, and runtime settings like `max_iterations` (default: 40) and `context_window_size`. You can edit it directly with `nvim ~/.config/tunacode.json` or see [the complete configuration example](documentation/configuration/config-file-example.md) for all available options.
 
 ### Recommended Models