tunacode-cli 0.1.21__py3-none-any.whl

tunacode/core/agents/prompts.py

@@ -0,0 +1,66 @@
+"""Prompt templates for agent intervention mechanisms.
+
+Extracted from main.py to centralize all prompt strings and formatting logic.
+"""
+
+
+def format_no_progress(
+    message: str,
+    unproductive_count: int,
+    last_productive: int,
+    current: int,
+    max_iterations: int,
+) -> str:
+    """Format the no-progress alert message.
+
+    Reference: main.py _force_action_if_unproductive() lines 265-275
+    """
+    return (
+        f"ALERT: No tools executed for {unproductive_count} iterations.\n\n"
+        f"Last productive iteration: {last_productive}\n"
+        f"Current iteration: {current}/{max_iterations}\n"
+        f"Task: {message[:200]}...\n\n"
+        "You're describing actions but not executing them. You MUST:\n\n"
+        "1. If task is COMPLETE: Start response with TUNACODE DONE:\n"
+        "2. If task needs work: Execute a tool RIGHT NOW (grep, read_file, bash, etc.)\n"
+        "3. If stuck: Explain the specific blocker\n\n"
+        "NO MORE DESCRIPTIONS. Take ACTION or mark COMPLETE."
+    )
+
+
+def format_clarification(original_query: str, iteration: int, tools_used: str) -> str:
+    """Format the clarification request message.
+
+    Reference: main.py _ask_for_clarification() lines 284-292
+    """
+    return (
+        "I need clarification to continue.\n\n"
+        f"Original request: {original_query}\n\n"
+        "Progress so far:\n"
+        f"- Iterations: {iteration}\n"
+        f"- Tools used: {tools_used}\n\n"
+        "If the task is complete, I should respond with TUNACODE DONE:\n"
+        "Otherwise, please provide specific guidance on what to do next."
+    )
+
+
+def format_iteration_limit(max_iterations: int, iteration: int, tools_used: str) -> str:
+    """Format the iteration limit reached message.
+
+    Reference: main.py process_request() lines 495-501
+    """
+    if tools_used == "No tools used yet":
+        tools_used = "No tools used"
+
+    return (
+        f"I've reached the iteration limit ({max_iterations}).\n\n"
+        "Progress summary:\n"
+        f"- Tools used: {tools_used}\n"
+        f"- Iterations completed: {iteration}\n\n"
+        "Please add more context to the task."
+    )
+
+
+# Note: Empty response handling is delegated to agent_components.handle_empty_response()
+# which uses create_empty_response_message() from agent_helpers.py
+# No template needed here as it's already modularized.

tunacode/core/agents/research_agent.py

@@ -0,0 +1,231 @@
+"""Research agent factory for read-only codebase exploration."""
+
+from pathlib import Path
+from typing import Any
+
+from httpx import AsyncClient, HTTPStatusError
+from pydantic_ai import Agent, Tool
+from pydantic_ai.retries import AsyncTenacityTransport, RetryConfig, wait_retry_after
+from tenacity import retry_if_exception_type, stop_after_attempt
+
+from tunacode.core.prompting import (
+    RESEARCH_TEMPLATE,
+    SectionLoader,
+    SystemPromptSection,
+    compose_prompt,
+    resolve_prompt,
+)
+from tunacode.core.state import StateManager
+from tunacode.tools.glob import glob
+from tunacode.tools.grep import grep
+from tunacode.tools.list_dir import list_dir
+from tunacode.tools.read_file import read_file
+from tunacode.types import ModelName, ToolProgress, ToolProgressCallback
+
+# Maximum wait time in seconds for retry backoff
+MAX_RETRY_WAIT_SECONDS = 60
+
+
+def _load_research_prompt() -> str:
+    """Load research-specific system prompt with section-based composition.
+
+    Loads individual section files from prompts/research/sections/ and
+    composes them using RESEARCH_TEMPLATE.
+
+    Returns:
+        Research system prompt content
+
+    Raises:
+        FileNotFoundError: If prompts/research/sections/ does not exist
+    """
+    # Navigate from this file: core/agents/research_agent.py -> src/tunacode/prompts/research
+    base_path = Path(__file__).parent.parent.parent
+    prompts_dir = base_path / "prompts" / "research"
+    sections_dir = prompts_dir / "sections"
+
+    if not sections_dir.exists():
+        raise FileNotFoundError(
+            f"Required sections directory not found: {sections_dir}. "
+            "The prompts/research/sections/ directory must exist."
+        )
+
+    loader = SectionLoader(sections_dir)
+    sections = {s.value: loader.load_section(s) for s in SystemPromptSection}
+
+    # Compose sections into research template, then resolve dynamic placeholders
+    prompt = compose_prompt(RESEARCH_TEMPLATE, sections)
+    return resolve_prompt(prompt)
+
+
+def _create_limited_read_file(max_files: int):
+    """Create a read_file wrapper that enforces a maximum number of calls.
+
+    Args:
+        max_files: Maximum number of files that can be read
+
+    Returns:
+        Wrapped read_file function with call limit enforcement
+    """
+    call_count = {"count": 0}
+
+    async def limited_read_file(file_path: str) -> str:
+        """Read file with enforced limit on number of calls.
+
+        Args:
+            file_path: Path to file to read
+
+        Returns:
+            File content dict from read_file tool, or limit message if exceeded
+
+        Note:
+            Returns a warning message instead of raising error when limit reached,
+            allowing the agent to complete with partial results.
+        """
+        if call_count["count"] >= max_files:
+            return (
+                "<file>\n"
+                f"FILE READ LIMIT REACHED ({max_files} files maximum)\n\n"
+                f"Cannot read '{file_path}' - you have already read {max_files} files.\n"
+                "Please complete your research with the files you have analyzed.\n"
+                "</file>"
+            )
+
+        call_count["count"] += 1
+        return await read_file(file_path)
+
+    return limited_read_file
+
+
+class ProgressTracker:
+    """Tracks tool execution progress for subagent feedback.
+
+    Note: total_operations is always 0 (unknown) since the number of
+    tool calls cannot be predicted upfront.
+    """
+
+    def __init__(self, callback: ToolProgressCallback | None, subagent_name: str = "research"):
+        self.callback = callback
+        self.subagent_name = subagent_name
+        self.operation_count = 0
+        self.total_operations = 0  # Always 0: total is unknown upfront
+
+    def emit(self, operation: str) -> None:
+        """Emit progress event for current operation."""
+        self.operation_count += 1
+        if self.callback:
+            progress = ToolProgress(
+                subagent=self.subagent_name,
+                operation=operation,
+                current=self.operation_count,
+                total=self.total_operations,
+            )
+            self.callback(progress)
+
+    def wrap_tool(self, tool_func, tool_name: str):
+        """Wrap a tool function to emit progress before execution."""
+
+        async def wrapped(*args, **kwargs) -> Any:
+            # Format operation description
+            if args:
+                first_arg = str(args[0])[:40]
+                operation = f"{tool_name} {first_arg}"
+            elif kwargs:
+                first_val = str(next(iter(kwargs.values())))[:40]
+                operation = f"{tool_name} {first_val}"
+            else:
+                operation = tool_name
+
+            self.emit(operation)
+            return await tool_func(*args, **kwargs)
+
+        # Preserve function metadata for pydantic-ai
+        wrapped.__name__ = tool_func.__name__
+        wrapped.__doc__ = tool_func.__doc__
+        wrapped.__annotations__ = getattr(tool_func, "__annotations__", {})
+
+        return wrapped
+
+
+def create_research_agent(
+    model: ModelName,
+    state_manager: StateManager,
+    max_files: int = 3,
+    progress_callback: ToolProgressCallback | None = None,
+) -> Agent[dict[str, Any]]:
+    """Create research agent with read-only tools and file read limit.
+
+    IMPORTANT: Uses same model as main agent - do NOT hardcode model selection.
+
+    Args:
+        model: The model name to use (same as main agent)
+        state_manager: State manager for session context
+        max_files: Maximum number of files the agent can read (hard limit, default: 3)
+        progress_callback: Optional callback for tool execution progress updates
+
+    Returns:
+        Agent configured with read-only tools, research system prompt, and file limit
+    """
+    # Load research-specific system prompt
+    system_prompt = _load_research_prompt()
+
+    # Get configuration from state manager
+    max_retries = state_manager.session.user_config.get("settings", {}).get("max_retries", 3)
+    tool_strict_validation = state_manager.session.user_config.get("settings", {}).get(
+        "tool_strict_validation", False
+    )
+
+    # Import here to avoid circular import with agent_config.py
+    # (agent_config imports delegation_tools which imports this module)
+    from tunacode.core.agents.agent_components.agent_config import (
+        _build_request_hooks,
+        _coerce_request_delay,
+        _create_model_with_retry,
+    )
+
+    transport = AsyncTenacityTransport(
+        config=RetryConfig(
+            retry=retry_if_exception_type(HTTPStatusError),
+            wait=wait_retry_after(max_wait=MAX_RETRY_WAIT_SECONDS),
+            stop=stop_after_attempt(max_retries),
+            reraise=True,
+        ),
+        validate_response=lambda r: r.raise_for_status(),
+    )
+    request_delay = _coerce_request_delay(state_manager)
+    event_hooks = _build_request_hooks(request_delay, state_manager)
+    http_client = AsyncClient(transport=transport, event_hooks=event_hooks)
+
+    model_instance = _create_model_with_retry(model, http_client, state_manager)
+
+    # Create limited read_file tool that enforces max_files cap
+    limited_read_file = _create_limited_read_file(max_files)
+
+    # Set up progress tracking if callback provided
+    tracker = ProgressTracker(progress_callback, "research")
+
+    # Wrap tools with progress tracking
+    if progress_callback:
+        tracked_read_file = tracker.wrap_tool(limited_read_file, "read_file")
+        tracked_grep = tracker.wrap_tool(grep, "grep")
+        tracked_list_dir = tracker.wrap_tool(list_dir, "list_dir")
+        tracked_glob = tracker.wrap_tool(glob, "glob")
+    else:
+        tracked_read_file = limited_read_file
+        tracked_grep = grep
+        tracked_list_dir = list_dir
+        tracked_glob = glob
+
+    # Create read-only tools list (no write/execute capabilities)
+    tools_list = [
+        Tool(tracked_read_file, max_retries=max_retries, strict=tool_strict_validation),
+        Tool(tracked_grep, max_retries=max_retries, strict=tool_strict_validation),
+        Tool(tracked_list_dir, max_retries=max_retries, strict=tool_strict_validation),
+        Tool(tracked_glob, max_retries=max_retries, strict=tool_strict_validation),
+    ]
+
+    return Agent(
+        model=model_instance,
+        system_prompt=system_prompt,
+        tools=tools_list,
+        output_type=dict,  # Structured research output as JSON dict
+    )

tunacode/core/compaction.py

@@ -0,0 +1,218 @@
+"""Tool output pruning for context window management.
+
+Implements backward-scanning algorithm to replace old tool outputs with placeholders,
+preserving conversation structure while freeing token budget.
+
+Inspired by OpenCode's compaction strategy.
+"""
+
+from typing import Any
+
+from tunacode.utils.messaging import estimate_tokens
+
+# Symbolic constants for pruning thresholds
+PRUNE_PROTECT_TOKENS: int = 40_000  # Protect last 40k tokens of tool outputs
+PRUNE_MINIMUM_THRESHOLD: int = 20_000  # Only prune if savings exceed 20k
+PRUNE_MIN_USER_TURNS: int = 2  # Require at least 2 user turns before pruning
+PRUNE_PLACEHOLDER: str = "[Old tool result content cleared]"
+
+# Message part kind identifiers
+PART_KIND_TOOL_RETURN: str = "tool-return"
+PART_KIND_USER_PROMPT: str = "user-prompt"
+
+
+def is_tool_return_part(part: Any) -> bool:
+    """Check if a message part is a tool return with content.
+
+    Args:
+        part: A message part object
+
+    Returns:
+        True if part has part_kind == "tool-return" and content attribute
+    """
+    if not hasattr(part, "part_kind"):
+        return False
+    if part.part_kind != PART_KIND_TOOL_RETURN:
+        return False
+    if not hasattr(part, "content"):  # noqa: SIM103
+        return False
+    return True
+
+
+def is_user_prompt_part(part: Any) -> bool:
+    """Check if a message part is a user prompt.
+
+    Args:
+        part: A message part object
+
+    Returns:
+        True if part has part_kind == "user-prompt"
+    """
+    if not hasattr(part, "part_kind"):
+        return False
+    return part.part_kind == PART_KIND_USER_PROMPT
+
+
+def count_user_turns(messages: list[Any]) -> int:
+    """Count the number of user message turns in history.
+
+    Counts messages containing UserPromptPart or dict messages with user content.
+
+    Args:
+        messages: Message history list
+
+    Returns:
+        Integer count of user turns
+    """
+    count = 0
+    for message in messages:
+        # Check for pydantic-ai message with parts
+        if hasattr(message, "parts"):
+            for part in message.parts:
+                if is_user_prompt_part(part):
+                    count += 1
+                    break  # Count each message only once
+        # Check for dict-style user message
+        elif isinstance(message, dict) and "content" in message:
+            role = message.get("role", "")
+            if role == "user":
+                count += 1
+    return count
+
+
+def estimate_part_tokens(part: Any, model_name: str) -> int:
+    """Estimate token count for a message part's content.
+
+    Args:
+        part: Message part with content attribute
+        model_name: Model for tiktoken encoding selection
+
+    Returns:
+        Estimated token count; 0 if content not extractable
+    """
+    if not hasattr(part, "content"):
+        return 0
+
+    content = part.content
+    if not isinstance(content, str):
+        # Non-string content, estimate based on repr
+        content = repr(content)
+
+    return estimate_tokens(content, model_name)
+
+
+def prune_part_content(part: Any, model_name: str) -> int:
+    """Replace a tool return part's content with placeholder.
+
+    Mutates the part in-place. Returns tokens reclaimed.
+
+    Args:
+        part: Tool return part to prune
+        model_name: Model for token estimation
+
+    Returns:
+        Number of tokens reclaimed (original - placeholder); 0 if cannot prune
+    """
+    if not hasattr(part, "content"):
+        return 0
+
+    content = part.content
+
+    # Skip already-pruned content
+    if content == PRUNE_PLACEHOLDER:
+        return 0
+
+    # Calculate original tokens
+    if isinstance(content, str):
+        original_tokens = estimate_tokens(content, model_name)
+    else:
+        original_tokens = estimate_tokens(repr(content), model_name)
+
+    # Calculate placeholder tokens
+    placeholder_tokens = estimate_tokens(PRUNE_PLACEHOLDER, model_name)
+
+    # Try to replace content
+    try:
+        part.content = PRUNE_PLACEHOLDER
+    except (AttributeError, TypeError):
+        # Part is immutable, cannot prune
+        return 0
+
+    return max(0, original_tokens - placeholder_tokens)
+
+
+def prune_old_tool_outputs(
+    messages: list[Any],
+    model_name: str,
+) -> tuple[list[Any], int]:
+    """Prune old tool output content from message history.
+
+    Scans message history backwards, protecting the most recent tool outputs
+    up to PRUNE_PROTECT_TOKENS, then replaces older tool output content
+    with PRUNE_PLACEHOLDER.
+
+    Args:
+        messages: List of pydantic-ai message objects (ModelRequest, ModelResponse, dict)
+        model_name: Model identifier for token estimation (e.g., "anthropic:claude-sonnet")
+
+    Returns:
+        Tuple of:
+            - Modified message list (same list, mutated in-place)
+            - Number of tokens reclaimed by pruning
+    """
+    if not messages:
+        return (messages, 0)
+
+    # Early exit: insufficient history
+    user_turns = count_user_turns(messages)
+    if user_turns < PRUNE_MIN_USER_TURNS:
+        return (messages, 0)
+
+    # Phase 1: Scan backwards, collect tool return parts with token counts
+    # Each entry: (message_index, part_index, part, token_count)
+    tool_parts: list[tuple[int, int, Any, int]] = []
+
+    for msg_idx in range(len(messages) - 1, -1, -1):
+        message = messages[msg_idx]
+        if not hasattr(message, "parts"):
+            continue
+
+        parts = message.parts
+        for part_idx in range(len(parts) - 1, -1, -1):
+            part = parts[part_idx]
+            if is_tool_return_part(part):
+                tokens = estimate_part_tokens(part, model_name)
+                tool_parts.append((msg_idx, part_idx, part, tokens))
+
+    if not tool_parts:
+        return (messages, 0)
+
+    # Phase 2: Determine pruning boundary
+    accumulated_tokens = 0
+    prune_start_index = -1
+
+    for i, (_, _, _, tokens) in enumerate(tool_parts):
+        accumulated_tokens += tokens
+        if accumulated_tokens > PRUNE_PROTECT_TOKENS:
+            prune_start_index = i
+            break
+
+    # Early exit: nothing old enough to prune
+    if prune_start_index < 0:
+        return (messages, 0)
+
+    # Phase 3: Calculate potential savings
+    parts_to_prune = tool_parts[prune_start_index:]
+    total_prunable_tokens = sum(tokens for _, _, _, tokens in parts_to_prune)
+
+    # Early exit: savings below threshold
+    if total_prunable_tokens < PRUNE_MINIMUM_THRESHOLD:
+        return (messages, 0)
+
+    # Phase 4: Apply pruning
+    total_reclaimed = 0
+    for _, _, part, _ in parts_to_prune:
+        reclaimed = prune_part_content(part, model_name)
+        total_reclaimed += reclaimed
+
+    return (messages, total_reclaimed)

tunacode/core/prompting/__init__.py

@@ -0,0 +1,27 @@
+"""Prompting engine for dynamic placeholder resolution and section composition."""
+
+from tunacode.core.prompting.loader import SectionLoader
+from tunacode.core.prompting.prompting_engine import (
+    PromptingEngine,
+    compose_prompt,
+    get_prompting_engine,
+    resolve_prompt,
+)
+from tunacode.core.prompting.sections import SystemPromptSection
+from tunacode.core.prompting.templates import (
+    MAIN_TEMPLATE,
+    RESEARCH_TEMPLATE,
+    TEMPLATE_OVERRIDES,
+)
+
+__all__ = [
+    "PromptingEngine",
+    "get_prompting_engine",
+    "resolve_prompt",
+    "compose_prompt",
+    "SystemPromptSection",
+    "MAIN_TEMPLATE",
+    "RESEARCH_TEMPLATE",
+    "TEMPLATE_OVERRIDES",
+    "SectionLoader",
+]

tunacode/core/prompting/loader.py

@@ -0,0 +1,66 @@
+"""Section loader for prompt templates."""
+
+from pathlib import Path
+
+from .sections import SystemPromptSection
+
+
+class SectionLoader:
+    """Loads prompt section content from files.
+
+    Supports .xml, .md, and .txt file extensions.
+    Uses mtime-based caching for efficiency.
+    """
+
+    EXTENSIONS = (".xml", ".md", ".txt")
+
+    def __init__(self, sections_dir: Path) -> None:
+        self.sections_dir = sections_dir
+        self._cache: dict[str, tuple[str, float]] = {}
+
+    def load_section(self, section: SystemPromptSection) -> str:
+        """Load a section's content from file.
+
+        Args:
+            section: The section to load
+
+        Returns:
+            Section content, or empty string if not found
+        """
+        filename = section.value.lower()
+        for ext in self.EXTENSIONS:
+            path = self.sections_dir / f"{filename}{ext}"
+            if path.exists():
+                return self._read_with_cache(path)
+        return ""
+
+    def load_all(self) -> dict[str, str]:
+        """Load all sections into a dict.
+
+        Returns:
+            Dict mapping section name to content
+        """
+        return {s.value: self.load_section(s) for s in SystemPromptSection}
+
+    def _read_with_cache(self, path: Path) -> str:
+        """Read file with mtime-based cache invalidation.
+
+        Args:
+            path: File path to read
+
+        Returns:
+            File contents
+        """
+        key = str(path)
+        mtime = path.stat().st_mtime
+        if key in self._cache:
+            cached_content, cached_mtime = self._cache[key]
+            if cached_mtime == mtime:
+                return cached_content
+        content = path.read_text()
+        self._cache[key] = (content, mtime)
+        return content
+
+    def clear_cache(self) -> None:
+        """Clear the file cache."""
+        self._cache.clear()