shotgun-sh 0.3.3.dev1__py3-none-any.whl → 0.6.2__py3-none-any.whl

shotgun/agents/config/provider.py

@@ -4,7 +4,7 @@ from pydantic import SecretStr
 from pydantic_ai.models import Model
 from pydantic_ai.models.anthropic import AnthropicModel, AnthropicModelSettings
 from pydantic_ai.models.google import GoogleModel
-from pydantic_ai.models.openai import OpenAIChatModel
+from pydantic_ai.models.openai import OpenAIResponsesModel
 from pydantic_ai.providers.anthropic import AnthropicProvider
 from pydantic_ai.providers.google import GoogleProvider
 from pydantic_ai.providers.openai import OpenAIProvider
@@ -47,18 +47,18 @@ def get_default_model_for_provider(config: ShotgunConfig) -> ModelName:
     """
     # Priority 1: Shotgun Account
     if _get_api_key(config.shotgun.api_key):
-        return ModelName.GPT_5_1
+        return ModelName.CLAUDE_SONNET_4_5
 
     # Priority 2: Individual provider keys
     if _get_api_key(config.anthropic.api_key):
-        return ModelName.CLAUDE_HAIKU_4_5
+        return ModelName.CLAUDE_SONNET_4_5
     if _get_api_key(config.openai.api_key):
-        return ModelName.GPT_5_1
+        return ModelName.GPT_5_2
     if _get_api_key(config.google.api_key):
-        return ModelName.GEMINI_2_5_PRO
+        return ModelName.GEMINI_3_PRO_PREVIEW
 
     # Fallback: system-wide default
-    return ModelName.CLAUDE_HAIKU_4_5
+    return ModelName.CLAUDE_SONNET_4_5
 
 
 def get_or_create_model(
@@ -130,7 +130,7 @@ def get_or_create_model(
                 # OpenAI and Google: Use LiteLLMProvider (OpenAI-compatible format)
                 # Google's GoogleProvider doesn't support base_url, so use LiteLLM
                 litellm_provider = create_litellm_provider(api_key)
-                _model_cache[cache_key] = OpenAIChatModel(
+                _model_cache[cache_key] = OpenAIResponsesModel(
                     litellm_model_name,
                     provider=litellm_provider,
                     settings=ModelSettings(max_tokens=max_tokens),
@@ -139,7 +139,7 @@ def get_or_create_model(
             # Use native provider implementations with user's API keys
             if provider == ProviderType.OPENAI:
                 openai_provider = OpenAIProvider(api_key=api_key)
-                _model_cache[cache_key] = OpenAIChatModel(
+                _model_cache[cache_key] = OpenAIResponsesModel(
                     model_name,
                     provider=openai_provider,
                     settings=ModelSettings(max_tokens=max_tokens),
@@ -257,23 +257,24 @@ async def get_provider_model(
         requested_model = None  # Will use provider's default model
 
     if provider_enum == ProviderType.OPENAI:
-        api_key = _get_api_key(config.openai.api_key)
+        api_key = _get_api_key(config.openai.api_key, "OPENAI_API_KEY")
         if not api_key:
-            raise ValueError("OpenAI API key not configured. Set via config.")
+            raise ValueError(
+                "OpenAI API key not configured. Set via config or OPENAI_API_KEY env var."
+            )
 
-        # Use requested model or default to gpt-5.1
-        model_name = requested_model if requested_model else ModelName.GPT_5_1
+        # Use requested model or default to gpt-5.2
+        model_name = requested_model if requested_model else ModelName.GPT_5_2
         # Gracefully fall back if model doesn't exist
         if model_name not in MODEL_SPECS:
-            model_name = ModelName.GPT_5_1
+            model_name = ModelName.GPT_5_2
         spec = MODEL_SPECS[model_name]
 
         # Check and test streaming capability for GPT-5 family models
         supports_streaming = True  # Default to True for all models
         if model_name in (
             ModelName.GPT_5_1,
-            ModelName.GPT_5_1_CODEX,
-            ModelName.GPT_5_1_CODEX_MINI,
+            ModelName.GPT_5_2,
         ):
             # Check if streaming capability has been tested
             streaming_capability = config.openai.supports_streaming
@@ -307,15 +308,17 @@ async def get_provider_model(
         )
 
     elif provider_enum == ProviderType.ANTHROPIC:
-        api_key = _get_api_key(config.anthropic.api_key)
+        api_key = _get_api_key(config.anthropic.api_key, "ANTHROPIC_API_KEY")
         if not api_key:
-            raise ValueError("Anthropic API key not configured. Set via config.")
+            raise ValueError(
+                "Anthropic API key not configured. Set via config or ANTHROPIC_API_KEY env var."
+            )
 
-        # Use requested model or default to claude-haiku-4-5
-        model_name = requested_model if requested_model else ModelName.CLAUDE_HAIKU_4_5
+        # Use requested model or default to claude-sonnet-4-5
+        model_name = requested_model if requested_model else ModelName.CLAUDE_SONNET_4_5
         # Gracefully fall back if model doesn't exist
         if model_name not in MODEL_SPECS:
-            model_name = ModelName.CLAUDE_HAIKU_4_5
+            model_name = ModelName.CLAUDE_SONNET_4_5
         spec = MODEL_SPECS[model_name]
 
         # Create fully configured ModelConfig
@@ -329,15 +332,19 @@ async def get_provider_model(
         )
 
     elif provider_enum == ProviderType.GOOGLE:
-        api_key = _get_api_key(config.google.api_key)
+        api_key = _get_api_key(config.google.api_key, "GEMINI_API_KEY")
         if not api_key:
-            raise ValueError("Gemini API key not configured. Set via config.")
+            raise ValueError(
+                "Gemini API key not configured. Set via config or GEMINI_API_KEY env var."
+            )
 
-        # Use requested model or default to gemini-2.5-pro
-        model_name = requested_model if requested_model else ModelName.GEMINI_2_5_PRO
+        # Use requested model or default to gemini-3-pro-preview
+        model_name = (
+            requested_model if requested_model else ModelName.GEMINI_3_PRO_PREVIEW
+        )
         # Gracefully fall back if model doesn't exist
         if model_name not in MODEL_SPECS:
-            model_name = ModelName.GEMINI_2_5_PRO
+            model_name = ModelName.GEMINI_3_PRO_PREVIEW
         spec = MODEL_SPECS[model_name]
 
         # Create fully configured ModelConfig
@@ -373,16 +380,26 @@ def _has_provider_key(config: "ShotgunConfig", provider: ProviderType) -> bool:
     return False
 
 
-def _get_api_key(config_key: SecretStr | None) -> str | None:
-    """Get API key from config.
+def _get_api_key(
+    config_key: SecretStr | None, env_var_name: str | None = None
+) -> str | None:
+    """Get API key from config or environment variable.
 
     Args:
         config_key: API key from configuration
+        env_var_name: Optional environment variable name to check as fallback
 
     Returns:
         API key string or None
     """
+    # First check config
     if config_key is not None:
         return config_key.get_secret_value()
 
+    # Fallback to environment variable
+    if env_var_name:
+        import os
+
+        return os.environ.get(env_var_name)
+
     return None

shotgun/agents/conversation/history/file_content_deduplication.py

@@ -5,8 +5,6 @@ tool returns before LLM-based compaction. Files are still accessible via
 `retrieve_code` (codebase) or `read_file` (.shotgun/ folder).
 """
 
-import copy
-import re
 from enum import StrEnum
 from typing import Any
 
@@ -43,40 +41,46 @@ SHOTGUN_PLACEHOLDER = (
     "**Content**: [Removed for compaction - file persisted in .shotgun/ folder]"
 )
 
-# Pattern for parsing file_read output (codebase files)
-# Format: **File**: `path`\n**Size**: N bytes\n[optional encoding]\n\n**Content**:\n```lang\ncontent```
-CODEBASE_FILE_PATTERN = re.compile(
-    r"\*\*File\*\*:\s*`([^`]+)`\s*\n"  # File path
-    r"\*\*Size\*\*:\s*(\d+)\s*bytes\s*\n"  # Size in bytes
-    r"(?:\*\*Encoding\*\*:.*?\n)?"  # Optional encoding line
-    r"\n\*\*Content\*\*:\s*\n"  # Blank line + Content header
-    r"```(\w*)\n"  # Language tag
-    r"(.*?)```",  # Actual content
-    re.DOTALL,
-)
+# Simple prefix for detecting file_read output format
+# Instead of using regex, we just check for the expected prefix and extract the file path
+CODEBASE_FILE_PREFIX = "**File**: `"
+
 
+def _extract_file_path(content: str) -> str | None:
+    """Extract file path from file_read tool return content.
 
-def _parse_codebase_file_content(
-    content: str,
-) -> tuple[str, int, str, str] | None:
-    """Parse file_read tool return content.
+    Uses simple string operations instead of regex for maximum performance.
+    The file_read tool output format is: **File**: `path`\\n...
 
     Args:
         content: The tool return content string
 
     Returns:
-        Tuple of (file_path, size_bytes, language, actual_content) or None if not parseable
+        The file path or None if format doesn't match
     """
-    match = CODEBASE_FILE_PATTERN.search(content)
-    if not match:
+    # Fast check: content must start with expected prefix
+    if not content.startswith(CODEBASE_FILE_PREFIX):
+        return None
+
+    # Find the closing backtick after the prefix
+    prefix_len = len(CODEBASE_FILE_PREFIX)
+    backtick_pos = content.find("`", prefix_len)
+
+    if backtick_pos == -1:
         return None
 
-    file_path = match.group(1)
-    size_bytes = int(match.group(2))
-    language = match.group(3) or ""
-    actual_content = match.group(4)
+    return content[prefix_len:backtick_pos]
+
+
+def _get_language_from_path(file_path: str) -> str:
+    """Infer programming language from file extension."""
+    from pathlib import Path
+
+    from shotgun.codebase.core.language_config import get_language_config
 
-    return file_path, size_bytes, language, actual_content
+    ext = Path(file_path).suffix
+    config = get_language_config(ext)
+    return config.name if config else "unknown"
 
 
 def _create_codebase_placeholder(file_path: str, size_bytes: int, language: str) -> str:
@@ -110,6 +114,11 @@ def deduplicate_file_content(
     This is a deterministic pre-compaction pass that reduces tokens without
     requiring an LLM. Files remain accessible via their respective tools.
 
+    This function uses copy-on-write semantics: only messages that need
+    modification are copied, while unmodified messages are reused by reference.
+    This significantly reduces memory allocation and processing time for large
+    conversations where only a subset of messages contain file content.
+
     Args:
         messages: Conversation history
         retention_window: Keep full content in last N messages (for recent context)
@@ -120,15 +129,17 @@ def deduplicate_file_content(
     if not messages:
         return messages, 0
 
-    # Deep copy to avoid modifying original
-    modified_messages = copy.deepcopy(messages)
     total_tokens_saved = 0
     files_deduplicated = 0
 
     # Calculate retention boundary (keep last N messages intact)
-    retention_start = max(0, len(modified_messages) - retention_window)
+    retention_start = max(0, len(messages) - retention_window)
+
+    # Track which message indices need replacement
+    # We use a dict to store index -> new_message mappings
+    replacements: dict[int, ModelMessage] = {}
 
-    for msg_idx, message in enumerate(modified_messages):
+    for msg_idx, message in enumerate(messages):
         # Skip messages in retention window
         if msg_idx >= retention_start:
             continue
@@ -159,18 +170,18 @@ def deduplicate_file_content(
 
             # Handle codebase file reads (file_read)
             if tool_name == FileReadTool.CODEBASE:
-                parsed = _parse_codebase_file_content(content)
-                if parsed:
-                    file_path, size_bytes, language, actual_content = parsed
-                    # Only replace if actual content is substantial
-                    if len(actual_content) >= MIN_CONTENT_LENGTH:
-                        replacement = _create_codebase_placeholder(
-                            file_path, size_bytes, language
-                        )
-                        logger.debug(
-                            f"Deduplicating codebase file: {file_path} "
-                            f"({size_bytes} bytes)"
-                        )
+                file_path = _extract_file_path(content)
+                if file_path:
+                    # Use content length as size estimate (includes formatting overhead
+                    # but close enough for deduplication purposes)
+                    size_bytes = len(content)
+                    language = _get_language_from_path(file_path)
+                    replacement = _create_codebase_placeholder(
+                        file_path, size_bytes, language
+                    )
+                    logger.debug(
+                        f"Deduplicating codebase file: {file_path} ({size_bytes} bytes)"
+                    )
 
             # Handle .shotgun/ file reads (read_file)
             elif tool_name == FileReadTool.SHOTGUN_FOLDER:
@@ -203,9 +214,21 @@ def deduplicate_file_content(
             else:
                 new_parts.append(part)
 
-        # Replace message with new parts if modified
+        # Only create a new message if parts were actually modified
         if message_modified:
-            modified_messages[msg_idx] = ModelRequest(parts=new_parts)
+            replacements[msg_idx] = ModelRequest(parts=new_parts)
+
+    # If no modifications were made, return original list (no allocation needed)
+    if not replacements:
+        return messages, 0
+
+    # Build result list with copy-on-write: reuse unmodified messages
+    modified_messages: list[ModelMessage] = []
+    for idx, msg in enumerate(messages):
+        if idx in replacements:
+            modified_messages.append(replacements[idx])
+        else:
+            modified_messages.append(msg)
 
     if files_deduplicated > 0:
         logger.info(

shotgun/agents/conversation/history/token_counting/base.py

@@ -2,7 +2,7 @@
 
 from abc import ABC, abstractmethod
 
-from pydantic_ai.messages import ModelMessage
+from pydantic_ai.messages import BinaryContent, ModelMessage
 
 
 class TokenCounter(ABC):
@@ -41,33 +41,75 @@ class TokenCounter(ABC):
         """
 
 
+def _extract_text_from_content(content: object) -> str | None:
+    """Extract text from a content object, skipping BinaryContent.
+
+    Args:
+        content: A content object (str, BinaryContent, list, etc.)
+
+    Returns:
+        Extracted text or None if content is binary/empty
+    """
+    if isinstance(content, BinaryContent):
+        return None
+    if isinstance(content, str):
+        return content.strip() if content.strip() else None
+    if isinstance(content, list):
+        # Content can be a list like ['text', BinaryContent(...), 'more text']
+        text_items = []
+        for item in content:
+            extracted = _extract_text_from_content(item)
+            if extracted:
+                text_items.append(extracted)
+        return "\n".join(text_items) if text_items else None
+    # For other types, convert to string but skip if it looks like binary
+    text = str(content)
+    # Skip if it's a BinaryContent repr (contains raw bytes)
+    if "BinaryContent(data=b" in text:
+        return None
+    return text.strip() if text.strip() else None
+
+
 def extract_text_from_messages(messages: list[ModelMessage]) -> str:
     """Extract all text content from messages for token counting.
 
+    Note: BinaryContent (PDFs, images) is skipped because:
+    1. str(BinaryContent) includes raw bytes which tokenize terribly
+    2. Claude uses fixed token costs for images/PDFs based on dimensions/pages,
+       not raw data size
+    3. Including binary data in text token counting causes massive overestimates
+       (e.g., 127KB of PDFs -> 267K tokens instead of ~few thousand)
+
     Args:
         messages: List of PydanticAI messages
 
     Returns:
-        Combined text content from all messages
+        Combined text content from all messages (excluding binary content)
     """
     text_parts = []
 
     for message in messages:
         if hasattr(message, "parts"):
             for part in message.parts:
-                if hasattr(part, "content") and isinstance(part.content, str):
-                    # Only add non-empty content
-                    if part.content.strip():
-                        text_parts.append(part.content)
+                # Skip BinaryContent directly
+                if isinstance(part, BinaryContent):
+                    continue
+
+                # Check if part has content attribute (UserPromptPart, etc.)
+                if hasattr(part, "content"):
+                    extracted = _extract_text_from_content(part.content)
+                    if extracted:
+                        text_parts.append(extracted)
                 else:
-                    # Handle non-text parts (tool calls, etc.)
+                    # Handle other parts (tool calls, etc.) - but check for binary
                     part_str = str(part)
-                    if part_str.strip():
+                    # Skip if it contains BinaryContent repr
+                    if "BinaryContent(data=b" not in part_str and part_str.strip():
                         text_parts.append(part_str)
         else:
             # Handle messages without parts
             msg_str = str(message)
-            if msg_str.strip():
+            if "BinaryContent(data=b" not in msg_str and msg_str.strip():
                 text_parts.append(msg_str)
 
     # If no valid text parts found, return a minimal placeholder

shotgun/agents/export.py

@@ -2,16 +2,15 @@
 
 from functools import partial
 
-from pydantic_ai import (
-    Agent,
-)
 from pydantic_ai.agent import AgentRunResult
 from pydantic_ai.messages import ModelMessage
 
 from shotgun.agents.config import ProviderType
+from shotgun.agents.models import ShotgunAgent
 from shotgun.logging_config import get_logger
 
 from .common import (
+    EventStreamHandler,
     add_system_status_message,
     build_agent_system_prompt,
     create_base_agent,
@@ -25,7 +24,7 @@ logger = get_logger(__name__)
 
 async def create_export_agent(
     agent_runtime_options: AgentRuntimeOptions, provider: ProviderType | None = None
-) -> tuple[Agent[AgentDeps, AgentResponse], AgentDeps]:
+) -> tuple[ShotgunAgent, AgentDeps]:
     """Create an export agent with file management capabilities.
 
     Args:
@@ -49,39 +48,39 @@ async def create_export_agent(
 
 
 async def run_export_agent(
-    agent: Agent[AgentDeps, AgentResponse],
-    instruction: str,
+    agent: ShotgunAgent,
+    prompt: str,
     deps: AgentDeps,
     message_history: list[ModelMessage] | None = None,
+    event_stream_handler: EventStreamHandler | None = None,
 ) -> AgentRunResult[AgentResponse]:
-    """Export artifacts based on the given instruction.
+    """Export artifacts based on the given prompt.
 
     Args:
         agent: The configured export agent
-        instruction: The export instruction
+        prompt: The export prompt
         deps: Agent dependencies
         message_history: Optional message history for conversation continuity
+        event_stream_handler: Optional callback for streaming events
 
     Returns:
         AgentRunResult containing the export process output
     """
-    logger.debug("📤 Starting export for instruction: %s", instruction)
+    logger.debug("📤 Starting export for prompt: %s", prompt)
 
     message_history = await add_system_status_message(deps, message_history)
 
-    # Let the agent use its tools to read existing artifacts and export them
-    full_prompt = f"Export artifacts or findings based on: {instruction}"
-
     try:
         # Create usage limits for responsible API usage
         usage_limits = create_usage_limits()
 
         result = await run_agent(
             agent=agent,
-            prompt=full_prompt,
+            prompt=prompt,
             deps=deps,
             message_history=message_history,
             usage_limits=usage_limits,
+            event_stream_handler=event_stream_handler,
         )
 
         logger.debug("✅ Export completed successfully")

shotgun/agents/file_read.py

@@ -0,0 +1,176 @@
+"""FileRead agent factory - lightweight agent for searching and reading files.
+
+This agent is designed for finding and reading files (including PDFs and images)
+without the overhead of full codebase understanding tools.
+"""
+
+from functools import partial
+
+from pydantic_ai import Agent, RunContext, UsageLimits
+from pydantic_ai.agent import AgentRunResult
+from pydantic_ai.messages import ModelMessage
+
+from shotgun.agents.config import ProviderType, get_provider_model
+from shotgun.agents.models import (
+    AgentDeps,
+    AgentResponse,
+    AgentRuntimeOptions,
+    AgentType,
+    ShotgunAgent,
+)
+from shotgun.logging_config import get_logger
+from shotgun.prompts import PromptLoader
+from shotgun.sdk.services import get_codebase_service
+from shotgun.utils import ensure_shotgun_directory_exists
+
+from .common import (
+    EventStreamHandler,
+    add_system_status_message,
+    run_agent,
+)
+from .conversation.history import token_limit_compactor
+from .tools import directory_lister, file_read, read_file
+from .tools.file_read_tools import multimodal_file_read
+
+logger = get_logger(__name__)
+
+# Prompt loader instance
+prompt_loader = PromptLoader()
+
+
+def _build_file_read_system_prompt(ctx: RunContext[AgentDeps]) -> str:
+    """Build system prompt for FileRead agent."""
+    template = prompt_loader.load_template("agents/file_read.j2")
+    return template.render(
+        interactive_mode=ctx.deps.interactive_mode,
+        mode="file_read",
+    )
+
+
+async def create_file_read_agent(
+    agent_runtime_options: AgentRuntimeOptions,
+    provider: ProviderType | None = None,
+) -> tuple[ShotgunAgent, AgentDeps]:
+    """Create a lightweight file reading agent.
+
+    This agent has minimal tools focused on file discovery and reading:
+    - directory_lister: List directory contents
+    - file_read: Read text files (from codebase tools)
+    - read_file: Read files by path
+    - multimodal_file_read: Read PDFs/images with BinaryContent
+
+    Args:
+        agent_runtime_options: Agent runtime options
+        provider: Optional provider override
+
+    Returns:
+        Tuple of (Configured agent, Agent dependencies)
+    """
+    logger.debug("Initializing FileRead agent")
+    ensure_shotgun_directory_exists()
+
+    # Get configured model
+    model_config = await get_provider_model(provider)
+    logger.debug(
+        "FileRead agent using %s model: %s",
+        model_config.provider.value.upper(),
+        model_config.name,
+    )
+
+    # Create minimal dependencies (no heavy codebase analysis)
+    codebase_service = get_codebase_service()
+
+    deps = AgentDeps(
+        **agent_runtime_options.model_dump(),
+        llm_model=model_config,
+        codebase_service=codebase_service,
+        system_prompt_fn=partial(_build_file_read_system_prompt),
+        agent_mode=AgentType.FILE_READ,
+    )
+
+    # History processor for context management
+    async def history_processor(messages: list[ModelMessage]) -> list[ModelMessage]:
+        class ProcessorContext:
+            def __init__(self, deps: AgentDeps):
+                self.deps = deps
+                self.usage = None
+
+        ctx = ProcessorContext(deps)
+        return await token_limit_compactor(ctx, messages)
+
+    # Create agent with structured output
+    model = model_config.model_instance
+    agent: ShotgunAgent = Agent(
+        model,
+        output_type=AgentResponse,
+        deps_type=AgentDeps,
+        instrument=True,
+        history_processors=[history_processor],
+        retries=3,
+    )
+
+    # Register only file reading tools (no write tools, no codebase query tools)
+    agent.tool(read_file)  # Basic file read
+    agent.tool(file_read)  # Codebase file read with CWD fallback
+    agent.tool(directory_lister)  # List directories
+    agent.tool(multimodal_file_read)  # PDF/image reading with BinaryContent
+
+    logger.debug("FileRead agent created with minimal tools")
+    return agent, deps
+
+
+def create_file_read_usage_limits() -> UsageLimits:
+    """Create conservative usage limits for FileRead agent.
+
+    FileRead should be quick - if it can't find the file in a few turns,
+    it should give up.
+    """
+    return UsageLimits(
+        request_limit=10,  # Max 10 API calls
+        request_tokens_limit=50_000,  # 50k input tokens
+        response_tokens_limit=8_000,  # 8k output tokens
+        total_tokens_limit=60_000,  # 60k total
+    )
+
+
+async def run_file_read_agent(
+    agent: ShotgunAgent,
+    prompt: str,
+    deps: AgentDeps,
+    message_history: list[ModelMessage] | None = None,
+    event_stream_handler: EventStreamHandler | None = None,
+) -> AgentRunResult[AgentResponse]:
+    """Run the FileRead agent to search for and read files.
+
+    Args:
+        agent: The configured FileRead agent
+        prompt: The file search prompt (e.g., "find the user stories PDF")
+        deps: Agent dependencies
+        message_history: Optional message history
+        event_stream_handler: Optional callback for streaming events
+
+    Returns:
+        AgentRunResult with response and files_found
+    """
+    logger.debug("FileRead agent searching: %s", prompt)
+
+    message_history = await add_system_status_message(deps, message_history)
+
+    try:
+        usage_limits = create_file_read_usage_limits()
+
+        result = await run_agent(
+            agent=agent,
+            prompt=prompt,
+            deps=deps,
+            message_history=message_history,
+            usage_limits=usage_limits,
+            event_stream_handler=event_stream_handler,
+        )
+
+        logger.debug("FileRead agent completed successfully")
+        return result
+
+    except Exception as e:
+        logger.error("FileRead agent failed: %s", str(e))
+        raise