shotgun-sh 0.1.0.dev13__py3-none-any.whl → 0.1.0.dev14__py3-none-any.whl

shotgun/agents/agent_manager.py

@@ -3,13 +3,19 @@
 from enum import Enum
 from typing import Any
 
-from pydantic_ai import Agent, DeferredToolRequests, DeferredToolResults, UsageLimits
+from pydantic_ai import (
+    Agent,
+    DeferredToolRequests,
+    DeferredToolResults,
+    UsageLimits,
+)
 from pydantic_ai.agent import AgentRunResult
 from pydantic_ai.messages import ModelMessage, ModelRequest
 from textual.message import Message
 from textual.widget import Widget
 
-from .models import AgentDeps, AgentRuntimeOptions
+from .history.compaction import apply_persistent_compaction
+from .models import AgentDeps, AgentRuntimeOptions, FileOperation
 from .plan import create_plan_agent
 from .research import create_research_agent
 from .tasks import create_tasks_agent
@@ -84,6 +90,7 @@ class AgentManager(Widget):
         # Maintain shared message history
         self.ui_message_history: list[ModelMessage] = []
         self.message_history: list[ModelMessage] = []
+        self.recently_change_files: list[FileOperation] = []
 
     @property
     def current_agent(self) -> Agent[AgentDeps, str | DeferredToolRequests]:
@@ -181,9 +188,15 @@ class AgentManager(Widget):
             mes for mes in result.new_messages() if not isinstance(mes, ModelRequest)
         ]
 
-        self.message_history = result.all_messages()
+        # Apply compaction to persistent message history to prevent cascading growth
+        self.message_history = await apply_persistent_compaction(
+            result.all_messages(), deps
+        )
         self._post_messages_updated()
 
+        # Log file operations summary if any files were modified
+        self.recently_change_files = deps.file_tracker.operations.copy()
+
         return result
 
     def _post_messages_updated(self) -> None:

shotgun/agents/artifact_state.py

@@ -0,0 +1,58 @@
+"""Utilities for collecting and organizing artifact state information."""
+
+from datetime import datetime
+from typing import TypedDict
+
+from shotgun.artifacts.models import ArtifactSummary
+from shotgun.artifacts.templates.models import TemplateSummary
+from shotgun.sdk.services import get_artifact_service
+
+
+class ArtifactState(TypedDict):
+    """Type definition for artifact state information."""
+
+    available_templates: dict[str, list[TemplateSummary]]
+    existing_artifacts: dict[str, list[ArtifactSummary]]
+    current_date: str
+
+
+def collect_artifact_state() -> ArtifactState:
+    """Collect and organize artifact state information for system context.
+
+    Returns:
+        ArtifactState containing organized templates and artifacts by mode, plus current date
+    """
+    artifact_service = get_artifact_service()
+
+    # Get available templates
+    available_templates_list = artifact_service.list_templates()
+
+    # Group templates by mode for better organization
+    templates_by_mode: dict[str, list[TemplateSummary]] = {}
+    for template in available_templates_list:
+        mode_name = template.template_id.split("/")[0]
+        if mode_name not in templates_by_mode:
+            templates_by_mode[mode_name] = []
+        templates_by_mode[mode_name].append(template)
+
+    # Get ALL existing artifacts regardless of current agent mode for complete visibility
+    existing_artifacts_list = (
+        artifact_service.list_artifacts()
+    )  # No mode filter = all modes
+
+    # Group artifacts by mode for organized display
+    artifacts_by_mode: dict[str, list[ArtifactSummary]] = {}
+    for artifact in existing_artifacts_list:
+        mode_name = artifact.agent_mode.value
+        if mode_name not in artifacts_by_mode:
+            artifacts_by_mode[mode_name] = []
+        artifacts_by_mode[mode_name].append(artifact)
+
+    # Get current date for temporal context (month in words for clarity)
+    current_date = datetime.now().strftime("%B %d, %Y")
+
+    return {
+        "available_templates": templates_by_mode,
+        "existing_artifacts": artifacts_by_mode,
+        "current_date": current_date,
+    }

shotgun/agents/common.py

@@ -23,10 +23,11 @@ from pydantic_ai.messages import (
 from shotgun.agents.config import ProviderType, get_config_manager, get_provider_model
 from shotgun.logging_config import get_logger
 from shotgun.prompts import PromptLoader
-from shotgun.sdk.services import get_codebase_service
+from shotgun.sdk.services import get_artifact_service, get_codebase_service
 from shotgun.utils import ensure_shotgun_directory_exists
 
 from .history import token_limit_compactor
+from .history.compaction import apply_persistent_compaction
 from .models import AgentDeps, AgentRuntimeOptions
 from .tools import (
     append_file,
@@ -70,10 +71,17 @@ async def add_system_status_message(
     message_history = message_history or []
     codebase_understanding_graphs = await deps.codebase_service.list_graphs()
 
+    # Collect artifact state information
+    from .artifact_state import collect_artifact_state
+
+    artifact_state = collect_artifact_state()
+
     system_state = prompt_loader.render(
         "agents/state/system_state.j2",
         codebase_understanding_graphs=codebase_understanding_graphs,
+        **artifact_state,
     )
+
     message_history.append(
         ModelResponse(
             parts=[
@@ -117,12 +125,14 @@ def create_base_agent(
         # Use the Model instance directly (has API key baked in)
         model = model_config.model_instance
 
-        # Create deps with model config and codebase service
+        # Create deps with model config and services
         codebase_service = get_codebase_service()
+        artifact_service = get_artifact_service()
         deps = AgentDeps(
             **agent_runtime_options.model_dump(),
             llm_model=model_config,
             codebase_service=codebase_service,
+            artifact_service=artifact_service,
             system_prompt_fn=system_prompt_fn,
         )
 
@@ -131,12 +141,25 @@ def create_base_agent(
         logger.debug("🤖 Creating agent with fallback OpenAI GPT-4o")
         raise ValueError("Configured model is required") from e
 
+    # Create a history processor that has access to deps via closure
+    async def history_processor(messages: list[ModelMessage]) -> list[ModelMessage]:
+        """History processor with access to deps via closure."""
+
+        # Create a minimal context for compaction
+        class ProcessorContext:
+            def __init__(self, deps: AgentDeps):
+                self.deps = deps
+                self.usage = None  # Will be estimated from messages
+
+        ctx = ProcessorContext(deps)
+        return await token_limit_compactor(ctx, messages)
+
     agent = Agent(
         model,
         output_type=[str, DeferredToolRequests],
         deps_type=AgentDeps,
         instrument=True,
-        history_processors=[token_limit_compactor],
+        history_processors=[history_processor],
     )
 
     # System prompt function is stored in deps and will be called manually in run_agent
@@ -153,17 +176,17 @@ def create_base_agent(
         logger.debug("📞 Interactive mode enabled - ask_user tool registered")
 
     # Register common file management tools (always available)
-    agent.tool_plain(read_file)
-    agent.tool_plain(write_file)
-    agent.tool_plain(append_file)
+    agent.tool(read_file)
+    agent.tool(write_file)
+    agent.tool(append_file)
 
     # Register artifact management tools (always available)
-    agent.tool_plain(create_artifact)
-    agent.tool_plain(list_artifacts)
-    agent.tool_plain(list_artifact_templates)
-    agent.tool_plain(read_artifact)
-    agent.tool_plain(read_artifact_section)
-    agent.tool_plain(write_artifact_section)
+    agent.tool(create_artifact)
+    agent.tool(list_artifacts)
+    agent.tool(list_artifact_templates)
+    agent.tool(read_artifact)
+    agent.tool(read_artifact_section)
+    agent.tool(write_artifact_section)
 
     # Register codebase understanding tools (conditional)
     if load_codebase_understanding_tools:
@@ -273,6 +296,10 @@ async def run_agent(
     message_history: list[ModelMessage] | None = None,
     usage_limits: UsageLimits | None = None,
 ) -> AgentRunResult[str | DeferredToolRequests]:
+    # Clear file tracker for new run
+    deps.file_tracker.clear()
+    logger.debug("🔧 Cleared file tracker for new agent run")
+
     # Add system prompt as first message
     message_history = await add_system_prompt_message(deps, message_history)
 
@@ -283,7 +310,8 @@ async def run_agent(
         message_history=message_history,
     )
 
-    messages = result.all_messages()
+    # Apply persistent compaction to prevent cascading token growth across CLI commands
+    messages = await apply_persistent_compaction(result.all_messages(), deps)
     while isinstance(result.output, DeferredToolRequests):
         logger.info("got deferred tool requests")
         await deps.queue.join()
@@ -306,6 +334,12 @@ async def run_agent(
             message_history=messages,
             deferred_tool_results=results,
         )
-        messages = result.all_messages()
+        # Apply persistent compaction to prevent cascading token growth in multi-turn loops
+        messages = await apply_persistent_compaction(result.all_messages(), deps)
+
+    # Log file operations summary if any files were modified
+    if deps.file_tracker.operations:
+        summary = deps.file_tracker.format_summary()
+        logger.info("📁 %s", summary)
 
     return result

shotgun/agents/config/models.py

@@ -1,9 +1,13 @@
 """Pydantic models for configuration."""
 
 from enum import Enum
+from typing import Any
 
 from pydantic import BaseModel, Field, PrivateAttr, SecretStr
+from pydantic_ai.direct import model_request
+from pydantic_ai.messages import ModelMessage, ModelResponse
 from pydantic_ai.models import Model
+from pydantic_ai.settings import ModelSettings
 
 
 class ProviderType(str, Enum):
@@ -57,6 +61,22 @@ class ModelConfig(BaseModel):
         }
         return f"{provider_prefix[self.provider]}:{self.name}"
 
+    def get_model_settings(self, max_tokens: int | None = None) -> ModelSettings:
+        """Get ModelSettings with optional token override.
+
+        This provides flexibility for specific use cases that need different
+        token limits while defaulting to maximum utilization.
+
+        Args:
+            max_tokens: Optional override for max_tokens. If None, uses max_output_tokens
+
+        Returns:
+            ModelSettings configured with specified or maximum tokens
+        """
+        return ModelSettings(
+            max_tokens=max_tokens if max_tokens is not None else self.max_output_tokens
+        )
+
 
 # Model specifications registry (static metadata)
 MODEL_SPECS: dict[str, ModelSpec] = {
@@ -125,3 +145,44 @@ class ShotgunConfig(BaseModel):
     )
     user_id: str = Field(description="Unique anonymous user identifier")
     config_version: int = Field(default=1, description="Configuration schema version")
+
+
+async def shotgun_model_request(
+    model_config: ModelConfig,
+    messages: list[ModelMessage],
+    max_tokens: int | None = None,
+    **kwargs: Any,
+) -> ModelResponse:
+    """Model request wrapper that uses full token capacity by default.
+
+    This wrapper ensures all LLM calls in Shotgun use the maximum available
+    token capacity of each model, improving response quality and completeness.
+    The most common issue this fixes is truncated summaries that were cut off
+    at default token limits (e.g., 4096 for Claude models).
+
+    Args:
+        model_config: ModelConfig instance with model settings and API key
+        messages: Messages to send to the model
+        max_tokens: Optional override for max_tokens. If None, uses model's max_output_tokens
+        **kwargs: Additional arguments passed to model_request
+
+    Returns:
+        ModelResponse from the model
+
+    Example:
+        # Uses full token capacity (e.g., 4096 for Claude, 128k for GPT-5)
+        response = await shotgun_model_request(model_config, messages)
+
+        # Override for specific use case
+        response = await shotgun_model_request(model_config, messages, max_tokens=1000)
+    """
+    # Get properly configured ModelSettings with maximum or overridden token limit
+    model_settings = model_config.get_model_settings(max_tokens)
+
+    # Make the model request with full token utilization
+    return await model_request(
+        model=model_config.model_instance,
+        messages=messages,
+        model_settings=model_settings,
+        **kwargs,
+    )

shotgun/agents/history/compaction.py

@@ -0,0 +1,85 @@
+"""Conversation compaction utilities."""
+
+from pydantic_ai.messages import ModelMessage
+from pydantic_ai.usage import RequestUsage
+
+from shotgun.agents.models import AgentDeps
+from shotgun.logging_config import get_logger
+
+from .token_estimation import estimate_tokens_from_messages
+
+logger = get_logger(__name__)
+
+
+async def apply_persistent_compaction(
+    messages: list[ModelMessage], deps: AgentDeps
+) -> list[ModelMessage]:
+    """Apply compaction to message history for persistent storage.
+
+    This ensures that compacted history is actually used as the conversation baseline,
+    preventing cascading compaction issues across both CLI and TUI usage patterns.
+
+    Args:
+        messages: Full message history from agent run
+        deps: Agent dependencies containing model config
+
+    Returns:
+        Compacted message history that should be stored as conversation state
+    """
+    from .history_processors import token_limit_compactor
+
+    try:
+        # Count actual token usage using shared utility
+        estimated_tokens = estimate_tokens_from_messages(messages, deps.llm_model)
+
+        # Create minimal usage info for compaction check
+        usage = RequestUsage(
+            input_tokens=estimated_tokens,
+            output_tokens=0,
+        )
+
+        # Create a minimal context object for compaction
+        class MockContext:
+            def __init__(self, deps: AgentDeps, usage: RequestUsage | None):
+                self.deps = deps
+                self.usage = usage
+
+        ctx = MockContext(deps, usage)
+        compacted_messages = await token_limit_compactor(ctx, messages)
+
+        # Log the result for monitoring
+        original_size = len(messages)
+        compacted_size = len(compacted_messages)
+
+        if compacted_size < original_size:
+            reduction_pct = ((original_size - compacted_size) / original_size) * 100
+            logger.debug(
+                f"Persistent compaction applied: {original_size} → {compacted_size} messages "
+                f"({reduction_pct:.1f}% reduction)"
+            )
+        else:
+            logger.debug(
+                f"No persistent compaction needed: {original_size} messages unchanged"
+            )
+
+        return compacted_messages
+
+    except Exception as e:
+        # If compaction fails, return original messages
+        # This ensures the system remains functional even if compaction has issues
+        logger.warning(f"Persistent compaction failed, using original history: {e}")
+        return messages
+
+
+def should_apply_persistent_compaction(deps: AgentDeps) -> bool:
+    """Check if persistent compaction should be applied.
+
+    Args:
+        deps: Agent dependencies
+
+    Returns:
+        True if persistent compaction should be applied
+    """
+    # For now, always apply persistent compaction
+    # Future: Add configuration option in deps or environment variable
+    return True

shotgun/agents/history/constants.py

@@ -0,0 +1,19 @@
+"""Constants for history processing and compaction."""
+
+from enum import Enum
+
+# Summary marker for compacted history
+SUMMARY_MARKER = "📌 COMPACTED_HISTORY:"
+
+# Token calculation constants
+INPUT_BUFFER_TOKENS = 500
+MIN_SUMMARY_TOKENS = 100
+TOKEN_LIMIT_RATIO = 0.8
+
+
+class SummaryType(Enum):
+    """Types of summarization requests for logging."""
+
+    INCREMENTAL = "INCREMENTAL"
+    FULL = "FULL"
+    CONTEXT_EXTRACTION = "CONTEXT_EXTRACTION"

shotgun/agents/history/context_extraction.py

@@ -0,0 +1,108 @@
+"""Context extraction utilities for history processing."""
+
+from pydantic_ai.messages import (
+    BuiltinToolCallPart,
+    BuiltinToolReturnPart,
+    ModelMessage,
+    ModelRequest,
+    ModelResponse,
+    ModelResponsePart,
+    RetryPromptPart,
+    SystemPromptPart,
+    TextPart,
+    ThinkingPart,
+    ToolCallPart,
+    ToolReturnPart,
+    UserPromptPart,
+)
+
+
+def extract_context_from_messages(messages: list[ModelMessage]) -> str:
+    """Extract context from a list of messages for summarization."""
+    context = ""
+    for msg in messages:
+        if isinstance(msg, ModelResponse | ModelRequest):
+            for part in msg.parts:
+                message_content = extract_context_from_part(part)
+                if message_content:
+                    context += message_content + "\n"
+    return context
+
+
+def extract_context_from_message_range(
+    messages: list[ModelMessage],
+    start_index: int,
+    end_index: int | None = None,
+) -> str:
+    """Extract context from a specific range of messages."""
+    if end_index is None:
+        end_index = len(messages)
+
+    message_slice = messages[start_index:end_index]
+    return extract_context_from_messages(message_slice)
+
+
+def has_meaningful_content(messages: list[ModelMessage]) -> bool:
+    """Check if messages contain meaningful content worth summarizing.
+
+    Only ModelResponse messages are considered meaningful for summarization.
+    User requests alone don't need summarization.
+    """
+    for msg in messages:
+        if isinstance(msg, ModelResponse):
+            for part in msg.parts:
+                if extract_context_from_part(part):
+                    return True
+    return False
+
+
+def extract_context_from_part(
+    message_part: (
+        SystemPromptPart
+        | UserPromptPart
+        | ToolReturnPart
+        | RetryPromptPart
+        | ModelResponsePart
+    ),
+) -> str:
+    """Extract context from a single message part."""
+    if isinstance(message_part, SystemPromptPart):
+        return ""  # Exclude system prompts from summary
+
+    elif isinstance(message_part, UserPromptPart):
+        if isinstance(message_part.content, str):
+            return f"<USER_PROMPT>\n{message_part.content}\n</USER_PROMPT>"
+        return ""
+
+    elif isinstance(message_part, ToolReturnPart):
+        return f"<TOOL_RETURN>\n{str(message_part.content)}\n</TOOL_RETURN>"
+
+    elif isinstance(message_part, RetryPromptPart):
+        if isinstance(message_part.content, str):
+            return f"<RETRY_PROMPT>\n{message_part.content}\n</RETRY_PROMPT>"
+        return ""
+
+    # Handle ModelResponsePart types
+    elif isinstance(message_part, TextPart):
+        return f"<ASSISTANT_TEXT>\n{message_part.content}\n</ASSISTANT_TEXT>"
+
+    elif isinstance(message_part, ToolCallPart):
+        if isinstance(message_part.args, dict):
+            args_str = ", ".join(f"{k}={repr(v)}" for k, v in message_part.args.items())
+            tool_call_str = f"{message_part.tool_name}({args_str})"
+        else:
+            tool_call_str = f"{message_part.tool_name}({message_part.args})"
+        return f"<TOOL_CALL>\n{tool_call_str}\n</TOOL_CALL>"
+
+    elif isinstance(message_part, BuiltinToolCallPart):
+        return f"<BUILTIN_TOOL_CALL>\n{message_part.tool_name}\n</BUILTIN_TOOL_CALL>"
+
+    elif isinstance(message_part, BuiltinToolReturnPart):
+        return (
+            f"<BUILTIN_TOOL_RETURN>\n{message_part.tool_name}\n</BUILTIN_TOOL_RETURN>"
+        )
+
+    elif isinstance(message_part, ThinkingPart):
+        return f"<THINKING>\n{message_part.content}\n</THINKING>"
+
+    return ""

shotgun/agents/history/history_building.py

@@ -0,0 +1,104 @@
+"""Functions for building compacted message history."""
+
+from pydantic_ai.messages import (
+    ModelMessage,
+    ModelRequest,
+    ModelRequestPart,
+    ModelResponse,
+    SystemPromptPart,
+    TextPart,
+    UserPromptPart,
+)
+
+from .message_utils import (
+    get_first_user_request,
+    get_last_user_request,
+    get_system_prompt,
+    get_user_content_from_request,
+)
+
+
+def build_clean_compacted_history(
+    summary_part: TextPart,
+    all_messages: list[ModelMessage],
+    last_summary_index: int | None = None,
+) -> list[ModelMessage]:
+    """Build a clean compacted history without preserving old verbose content.
+
+    Args:
+        summary_part: The marked summary part to include
+        all_messages: Original message history
+        last_summary_index: Index of the last summary (if any)
+
+    Returns:
+        Clean compacted message history
+    """
+    # Extract essential context from pre-summary messages (if any)
+    system_prompt = ""
+    first_user_prompt = ""
+
+    if last_summary_index is not None and last_summary_index > 0:
+        # Get system and first user from original conversation
+        pre_summary_messages = all_messages[:last_summary_index]
+        system_prompt = get_system_prompt(pre_summary_messages) or ""
+        first_user_prompt = get_first_user_request(pre_summary_messages) or ""
+
+    # Build the base structure
+    compacted_messages: list[ModelMessage] = []
+
+    # Add system/user context if it exists and is meaningful
+    if system_prompt or first_user_prompt:
+        compacted_messages.append(
+            _create_base_request(system_prompt, first_user_prompt)
+        )
+
+    # Add the summary
+    summary_message = ModelResponse(parts=[summary_part])
+    compacted_messages.append(summary_message)
+
+    # Ensure proper ending
+    return ensure_ends_with_model_request(compacted_messages, all_messages)
+
+
+def ensure_ends_with_model_request(
+    compacted_messages: list[ModelMessage],
+    original_messages: list[ModelMessage],
+) -> list[ModelMessage]:
+    """Ensure the message history ends with ModelRequest for PydanticAI compatibility."""
+    last_user_request = get_last_user_request(original_messages)
+
+    if not last_user_request:
+        return compacted_messages
+
+    # Check if we need to add the last request or restructure
+    if compacted_messages and isinstance(compacted_messages[0], ModelRequest):
+        first_request = compacted_messages[0]
+        last_user_content = get_user_content_from_request(last_user_request)
+        first_user_content = get_user_content_from_request(first_request)
+
+        if last_user_content != first_user_content:
+            # Different messages - append the last request
+            compacted_messages.append(last_user_request)
+        else:
+            # Same message - restructure to end with ModelRequest
+            if len(compacted_messages) >= 2:
+                summary_message = compacted_messages[1]  # The summary
+                compacted_messages = [summary_message, first_request]
+    else:
+        # No first request, just add the last one
+        compacted_messages.append(last_user_request)
+
+    return compacted_messages
+
+
+def _create_base_request(system_prompt: str, user_prompt: str) -> ModelRequest:
+    """Create the base ModelRequest with system and user prompts."""
+    parts: list[ModelRequestPart] = []
+
+    if system_prompt:
+        parts.append(SystemPromptPart(content=system_prompt))
+
+    if user_prompt:
+        parts.append(UserPromptPart(content=user_prompt))
+
+    return ModelRequest(parts=parts)