shotgun-sh 0.2.3.dev2__py3-none-any.whl → 0.2.11.dev5__py3-none-any.whl

shotgun/agents/tools/web_search/gemini.py

@@ -8,11 +8,22 @@ from shotgun.agents.config import get_provider_model
 from shotgun.agents.config.constants import MEDIUM_TEXT_8K_TOKENS
 from shotgun.agents.config.models import ModelName
 from shotgun.agents.llm import shotgun_model_request
+from shotgun.agents.tools.registry import ToolCategory, register_tool
 from shotgun.logging_config import get_logger
+from shotgun.prompts import PromptLoader
+from shotgun.utils.datetime_utils import get_datetime_context
 
 logger = get_logger(__name__)
 
+# Global prompt loader instance
+prompt_loader = PromptLoader()
 
+
+@register_tool(
+    category=ToolCategory.WEB_RESEARCH,
+    display_text="Searching web",
+    key_arg="query",
+)
 async def gemini_web_search_tool(query: str) -> str:
     """Perform a web search using Google's Gemini API with grounding.
 
@@ -35,23 +46,24 @@ async def gemini_web_search_tool(query: str) -> str:
 
     # Get model configuration (supports both Shotgun and BYOK)
     try:
-        model_config = get_provider_model(ModelName.GEMINI_2_5_FLASH)
+        model_config = await get_provider_model(ModelName.GEMINI_2_5_FLASH)
     except ValueError as e:
         error_msg = f"Gemini API key not configured: {str(e)}"
         logger.error("❌ %s", error_msg)
         span.set_attribute("output.value", f"**Error:**\n {error_msg}\n")
         return error_msg
 
-    # Create a search-optimized prompt
-    search_prompt = f"""Please provide current and accurate information about the following query:
-
-Query: {query}
-
-Instructions:
-- Provide comprehensive, factual information
-- Include relevant details and context
-- Focus on current and recent information
-- Be specific and accurate in your response"""
+    # Get datetime context for the search prompt
+    dt_context = get_datetime_context()
+
+    # Render search prompt from template
+    search_prompt = prompt_loader.render(
+        "tools/web_search.j2",
+        query=query,
+        current_datetime=dt_context.datetime_formatted,
+        timezone_name=dt_context.timezone_name,
+        utc_offset=dt_context.utc_offset,
+    )
 
     # Build the request messages
     messages: list[ModelMessage] = [ModelRequest.user_text_prompt(search_prompt)]

shotgun/agents/tools/web_search/openai.py

@@ -5,11 +5,22 @@ from opentelemetry import trace
 
 from shotgun.agents.config import get_provider_model
 from shotgun.agents.config.models import ProviderType
+from shotgun.agents.tools.registry import ToolCategory, register_tool
 from shotgun.logging_config import get_logger
+from shotgun.prompts import PromptLoader
+from shotgun.utils.datetime_utils import get_datetime_context
 
 logger = get_logger(__name__)
 
+# Global prompt loader instance
+prompt_loader = PromptLoader()
 
+
+@register_tool(
+    category=ToolCategory.WEB_RESEARCH,
+    display_text="Searching web",
+    key_arg="query",
+)
 async def openai_web_search_tool(query: str) -> str:
     """Perform a web search and return results.
 
@@ -32,7 +43,7 @@ async def openai_web_search_tool(query: str) -> str:
 
         # Get API key from centralized configuration
         try:
-            model_config = get_provider_model(ProviderType.OPENAI)
+            model_config = await get_provider_model(ProviderType.OPENAI)
             api_key = model_config.api_key
         except ValueError as e:
             error_msg = f"OpenAI API key not configured: {str(e)}"
@@ -40,19 +51,17 @@ async def openai_web_search_tool(query: str) -> str:
             span.set_attribute("output.value", f"**Error:**\n {error_msg}\n")
             return error_msg
 
-        prompt = f"""Please provide current and accurate information about the following query:
-
-Query: {query}
+        # Get datetime context for the search prompt
+        dt_context = get_datetime_context()
 
-Instructions:
-- Provide comprehensive, factual information
-- Include relevant details and context
-- Focus on current and recent information
-- Be specific and accurate in your response
-- You can't ask the user for details, so assume the most relevant details for the query
-
-ALWAYS PROVIDE THE SOURCES (urls) TO BACK UP THE INFORMATION YOU PROVIDE.
-"""
+        # Render search prompt from template
+        prompt = prompt_loader.render(
+            "tools/web_search.j2",
+            query=query,
+            current_datetime=dt_context.datetime_formatted,
+            timezone_name=dt_context.timezone_name,
+            utc_offset=dt_context.utc_offset,
+        )
 
         client = AsyncOpenAI(api_key=api_key)
         response = await client.responses.create(  # type: ignore[call-overload]

shotgun/agents/tools/web_search/utils.py

@@ -4,7 +4,7 @@ from shotgun.agents.config import get_provider_model
 from shotgun.agents.config.models import ProviderType
 
 
-def is_provider_available(provider: ProviderType) -> bool:
+async def is_provider_available(provider: ProviderType) -> bool:
     """Check if a provider has API key configured.
 
     Args:
@@ -14,7 +14,7 @@ def is_provider_available(provider: ProviderType) -> bool:
         True if the provider has valid credentials configured (from config or env)
     """
     try:
-        get_provider_model(provider)
+        await get_provider_model(provider)
         return True
     except ValueError:
         return False

shotgun/agents/usage_manager.py

@@ -6,6 +6,8 @@ from logging import getLogger
 from pathlib import Path
 from typing import TypeAlias
 
+import aiofiles
+import aiofiles.os
 from genai_prices import calc_price
 from pydantic import BaseModel, Field
 from pydantic_ai import RunUsage
@@ -48,9 +50,10 @@ class SessionUsageManager:
         self._model_providers: dict[ModelName, ProviderType] = {}
         self._usage_log: list[UsageLogEntry] = []
         self._usage_path: Path = get_shotgun_home() / "usage.json"
-        self.restore_usage_state()
+        # Note: restore_usage_state needs to be called asynchronously after init
+        # Caller should use: manager = SessionUsageManager(); await manager.restore_usage_state()
 
-    def add_usage(
+    async def add_usage(
         self, usage: RunUsage, *, model_name: ModelName, provider: ProviderType
     ) -> None:
         self.usage[model_name] += usage
@@ -58,7 +61,7 @@ class SessionUsageManager:
         self._usage_log.append(
             UsageLogEntry(model_name=model_name, usage=usage, provider=provider)
         )
-        self.persist_usage_state()
+        await self.persist_usage_state()
 
     def get_usage_report(self) -> dict[ModelName, RunUsage]:
         return self.usage.copy()
@@ -78,7 +81,7 @@ class SessionUsageManager:
     def build_usage_hint(self) -> str | None:
         return format_usage_hint(self.get_usage_breakdown())
 
-    def persist_usage_state(self) -> None:
+    async def persist_usage_state(self) -> None:
         state = UsageState(
             usage=dict(self.usage.items()),
             model_providers=self._model_providers.copy(),
@@ -86,23 +89,25 @@ class SessionUsageManager:
         )
 
         try:
-            self._usage_path.parent.mkdir(parents=True, exist_ok=True)
-            with self._usage_path.open("w", encoding="utf-8") as f:
-                json.dump(state.model_dump(mode="json"), f, indent=2)
+            await aiofiles.os.makedirs(self._usage_path.parent, exist_ok=True)
+            json_content = json.dumps(state.model_dump(mode="json"), indent=2)
+            async with aiofiles.open(self._usage_path, "w", encoding="utf-8") as f:
+                await f.write(json_content)
             logger.debug("Usage state persisted to %s", self._usage_path)
         except Exception as exc:
             logger.error(
                 "Failed to persist usage state to %s: %s", self._usage_path, exc
             )
 
-    def restore_usage_state(self) -> None:
-        if not self._usage_path.exists():
+    async def restore_usage_state(self) -> None:
+        if not await aiofiles.os.path.exists(self._usage_path):
             logger.debug("No usage state file found at %s", self._usage_path)
             return
 
         try:
-            with self._usage_path.open(encoding="utf-8") as f:
-                data = json.load(f)
+            async with aiofiles.open(self._usage_path, encoding="utf-8") as f:
+                content = await f.read()
+                data = json.loads(content)
 
             state = UsageState.model_validate(data)
         except Exception as exc:

shotgun/api_endpoints.py

@@ -1,10 +1,14 @@
 """Shotgun backend service API endpoints and URLs."""
 
+from shotgun.settings import settings
+
 # Shotgun Web API base URL (for authentication/subscription)
-# Can be overridden with environment variable
-SHOTGUN_WEB_BASE_URL = "https://api-219702594231.us-east4.run.app"
+# Can be overridden with SHOTGUN_WEB_BASE_URL environment variable
+SHOTGUN_WEB_BASE_URL = settings.api.web_base_url
+
 # Shotgun's LiteLLM proxy base URL (for AI model requests)
-LITELLM_PROXY_BASE_URL = "https://litellm-219702594231.us-east4.run.app"
+# Can be overridden with SHOTGUN_ACCOUNT_LLM_BASE_URL environment variable
+LITELLM_PROXY_BASE_URL = settings.api.account_llm_base_url
 
 # Provider-specific LiteLLM proxy endpoints
 LITELLM_PROXY_ANTHROPIC_BASE = f"{LITELLM_PROXY_BASE_URL}/anthropic"

shotgun/build_constants.py

@@ -13,7 +13,7 @@ POSTHOG_PROJECT_ID = '191396'
 
 # Logfire configuration embedded at build time (only for dev builds)
 LOGFIRE_ENABLED = 'true'
-LOGFIRE_TOKEN = 'pylf_v1_us_KZ5NM1pP3NwgJkbBJt6Ftdzk8mMhmrXcGJHQQgDJ1LfK'
+LOGFIRE_TOKEN = 'pylf_v1_us_RwZMlJm1tX6j0PL5RWWbmZpzK2hLBNtFWStNKlySfjh8'
 
 # Build metadata
 BUILD_TIME_ENV = "production" if SENTRY_DSN else "development"

shotgun/cli/clear.py

@@ -0,0 +1,53 @@
+"""Clear command for shotgun CLI."""
+
+import asyncio
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from shotgun.agents.conversation_manager import ConversationManager
+from shotgun.logging_config import get_logger
+
+app = typer.Typer(
+    name="clear", help="Clear the conversation history", no_args_is_help=False
+)
+logger = get_logger(__name__)
+console = Console()
+
+
+@app.callback(invoke_without_command=True)
+def clear() -> None:
+    """Clear the current conversation history.
+
+    This command deletes the conversation file at ~/.shotgun-sh/conversation.json,
+    removing all conversation history. Other files in ~/.shotgun-sh/ (config, usage,
+    codebases, logs) are preserved.
+    """
+    try:
+        # Get conversation file path
+        conversation_file = Path.home() / ".shotgun-sh" / "conversation.json"
+
+        # Check if file exists
+        if not conversation_file.exists():
+            console.print(
+                "[yellow]No conversation file found.[/yellow] Nothing to clear.",
+                style="bold",
+            )
+            return
+
+        # Clear the conversation
+        manager = ConversationManager(conversation_file)
+        asyncio.run(manager.clear())
+
+        console.print(
+            "[green]✓[/green] Conversation cleared successfully", style="bold"
+        )
+        logger.info("Conversation cleared successfully")
+
+    except Exception as e:
+        console.print(
+            f"[red]Error:[/red] Failed to clear conversation: {e}", style="bold"
+        )
+        logger.debug("Full traceback:", exc_info=True)
+        raise typer.Exit(code=1) from e

shotgun/cli/compact.py

@@ -0,0 +1,186 @@
+"""Compact command for shotgun CLI."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from pydantic_ai.usage import RequestUsage
+from rich.console import Console
+
+from shotgun.agents.config import get_provider_model
+from shotgun.agents.conversation_manager import ConversationManager
+from shotgun.agents.history.history_processors import token_limit_compactor
+from shotgun.agents.history.token_estimation import estimate_tokens_from_messages
+from shotgun.cli.models import OutputFormat
+from shotgun.logging_config import get_logger
+
+app = typer.Typer(
+    name="compact", help="Compact the conversation history", no_args_is_help=False
+)
+logger = get_logger(__name__)
+console = Console()
+
+
+@app.callback(invoke_without_command=True)
+def compact(
+    format: Annotated[
+        OutputFormat,
+        typer.Option(
+            "--format",
+            "-f",
+            help="Output format: markdown or json",
+        ),
+    ] = OutputFormat.MARKDOWN,
+) -> None:
+    """Compact the current conversation history to reduce size.
+
+    This command compacts the conversation in ~/.shotgun-sh/conversation.json
+    by summarizing older messages while preserving recent context. The compacted
+    conversation is automatically saved back to the file.
+    """
+    try:
+        result = asyncio.run(compact_conversation())
+
+        if format == OutputFormat.JSON:
+            # Output as JSON
+            console.print_json(json.dumps(result, indent=2))
+        else:
+            # Output as markdown
+            console.print(format_markdown(result))
+
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error:[/red] {e}\n\n"
+            "No conversation found. Start a TUI session first with: [cyan]shotgun[/cyan]",
+            style="bold",
+        )
+        raise typer.Exit(code=1) from e
+    except Exception as e:
+        console.print(
+            f"[red]Error:[/red] Failed to compact conversation: {e}", style="bold"
+        )
+        logger.debug("Full traceback:", exc_info=True)
+        raise typer.Exit(code=1) from e
+
+
+async def compact_conversation() -> dict[str, Any]:
+    """Compact the conversation and return statistics.
+
+    Returns:
+        Dictionary with compaction statistics including before/after metrics
+    """
+    # Get conversation file path
+    conversation_file = Path.home() / ".shotgun-sh" / "conversation.json"
+
+    if not conversation_file.exists():
+        raise FileNotFoundError(f"Conversation file not found at {conversation_file}")
+
+    # Load conversation
+    manager = ConversationManager(conversation_file)
+    conversation = await manager.load()
+
+    if not conversation:
+        raise ValueError("Conversation file is empty or corrupted")
+
+    # Get agent messages only (not UI messages)
+    agent_messages = conversation.get_agent_messages()
+
+    if not agent_messages:
+        raise ValueError("No agent messages found in conversation")
+
+    # Get model config
+    model_config = await get_provider_model()
+
+    # Calculate before metrics
+    original_message_count = len(agent_messages)
+    original_tokens = await estimate_tokens_from_messages(agent_messages, model_config)
+
+    # For CLI, we can call token_limit_compactor directly without full AgentDeps
+    # since we only need the model config and message history
+    # Create a minimal context object for compaction
+    class CompactContext:
+        def __init__(self, model_config: Any, usage: RequestUsage) -> None:
+            self.deps = type("Deps", (), {"llm_model": model_config})()
+            self.usage = usage
+
+    # Create minimal usage info for compaction check
+    usage = RequestUsage(input_tokens=original_tokens, output_tokens=0)
+    ctx = CompactContext(model_config, usage)
+
+    # Apply compaction with force=True to bypass threshold checks
+    compacted_messages = await token_limit_compactor(ctx, agent_messages, force=True)
+
+    # Calculate after metrics
+    compacted_message_count = len(compacted_messages)
+    compacted_tokens = await estimate_tokens_from_messages(
+        compacted_messages, model_config
+    )
+
+    # Calculate reduction percentages
+    message_reduction = (
+        ((original_message_count - compacted_message_count) / original_message_count)
+        * 100
+        if original_message_count > 0
+        else 0
+    )
+    token_reduction = (
+        ((original_tokens - compacted_tokens) / original_tokens) * 100
+        if original_tokens > 0
+        else 0
+    )
+
+    # Save compacted conversation
+    conversation.set_agent_messages(compacted_messages)
+    await manager.save(conversation)
+
+    logger.info(
+        f"Compacted conversation: {original_message_count} → {compacted_message_count} messages "
+        f"({message_reduction:.1f}% reduction)"
+    )
+
+    return {
+        "success": True,
+        "before": {
+            "messages": original_message_count,
+            "estimated_tokens": original_tokens,
+        },
+        "after": {
+            "messages": compacted_message_count,
+            "estimated_tokens": compacted_tokens,
+        },
+        "reduction": {
+            "messages_percent": round(message_reduction, 1),
+            "tokens_percent": round(token_reduction, 1),
+        },
+    }
+
+
+def format_markdown(result: dict[str, Any]) -> str:
+    """Format compaction result as markdown.
+
+    Args:
+        result: Dictionary with compaction statistics
+
+    Returns:
+        Formatted markdown string
+    """
+    before = result["before"]
+    after = result["after"]
+    reduction = result["reduction"]
+
+    return f"""# Conversation Compacted ✓
+
+## Before
+- **Messages:** {before["messages"]:,}
+- **Estimated Tokens:** {before["estimated_tokens"]:,}
+
+## After
+- **Messages:** {after["messages"]:,}
+- **Estimated Tokens:** {after["estimated_tokens"]:,}
+
+## Reduction
+- **Messages:** {reduction["messages_percent"]}%
+- **Tokens:** {reduction["tokens_percent"]}%
+"""

shotgun/cli/config.py

@@ -1,5 +1,6 @@
 """Configuration management CLI commands."""
 
+import asyncio
 import json
 from typing import Annotated, Any
 
@@ -44,7 +45,7 @@ def init(
         console.print()
 
         # Initialize with defaults
-        config_manager.initialize()
+        asyncio.run(config_manager.initialize())
 
         # Ask for provider
         provider_choices = ["openai", "anthropic", "google"]
@@ -76,7 +77,7 @@ def init(
 
         if api_key:
             # update_provider will automatically set selected_model for first provider
-            config_manager.update_provider(provider, api_key=api_key)
+            asyncio.run(config_manager.update_provider(provider, api_key=api_key))
 
         console.print(
             f"\n✅ [bold green]Configuration saved to {config_manager.config_path}[/bold green]"
@@ -84,7 +85,7 @@ def init(
         console.print("🎯 You can now use Shotgun with your configured provider!")
 
     else:
-        config_manager.initialize()
+        asyncio.run(config_manager.initialize())
         console.print(f"✅ Configuration initialized at {config_manager.config_path}")
 
 
@@ -112,7 +113,7 @@ def set(
 
     try:
         if api_key:
-            config_manager.update_provider(provider, api_key=api_key)
+            asyncio.run(config_manager.update_provider(provider, api_key=api_key))
 
         console.print(f"✅ Configuration updated for {provider}")
 
@@ -133,8 +134,10 @@ def get(
     ] = False,
 ) -> None:
     """Display current configuration."""
+    import asyncio
+
     config_manager = get_config_manager()
-    config = config_manager.load()
+    config = asyncio.run(config_manager.load())
 
     if json_output:
         # Convert to dict and mask secrets

shotgun/cli/context.py

@@ -0,0 +1,111 @@
+"""Context command for shotgun CLI."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from shotgun.agents.config import get_provider_model
+from shotgun.agents.context_analyzer import (
+    ContextAnalysisOutput,
+    ContextAnalyzer,
+    ContextFormatter,
+)
+from shotgun.agents.conversation_manager import ConversationManager
+from shotgun.cli.models import OutputFormat
+from shotgun.logging_config import get_logger
+
+app = typer.Typer(
+    name="context", help="Analyze conversation context usage", no_args_is_help=False
+)
+logger = get_logger(__name__)
+console = Console()
+
+
+@app.callback(invoke_without_command=True)
+def context(
+    format: Annotated[
+        OutputFormat,
+        typer.Option(
+            "--format",
+            "-f",
+            help="Output format: markdown or json",
+        ),
+    ] = OutputFormat.MARKDOWN,
+) -> None:
+    """Analyze the current conversation's context usage.
+
+    This command analyzes the agent's message history from ~/.shotgun-sh/conversation.json
+    and displays token usage breakdown by message type. Only agent context is counted
+    (UI elements like hints are excluded).
+    """
+    try:
+        result = asyncio.run(analyze_context())
+
+        if format == OutputFormat.JSON:
+            # Output as JSON
+            console.print_json(json.dumps(result.json_data, indent=2))
+        else:
+            # Output as plain text (Markdown() reformats and makes categories inline)
+            console.print(result.markdown)
+
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error:[/red] {e}\n\n"
+            "No conversation found. Start a TUI session first with: [cyan]shotgun[/cyan]",
+            style="bold",
+        )
+        raise typer.Exit(code=1) from e
+    except Exception as e:
+        console.print(f"[red]Error:[/red] Failed to analyze context: {e}", style="bold")
+        logger.debug("Full traceback:", exc_info=True)
+        raise typer.Exit(code=1) from e
+
+
+async def analyze_context() -> ContextAnalysisOutput:
+    """Analyze the conversation context and return structured data.
+
+    Returns:
+        ContextAnalysisOutput with both markdown and JSON representations of the analysis
+    """
+    # Get conversation file path
+    conversation_file = Path.home() / ".shotgun-sh" / "conversation.json"
+
+    if not conversation_file.exists():
+        raise FileNotFoundError(f"Conversation file not found at {conversation_file}")
+
+    # Load conversation
+    manager = ConversationManager(conversation_file)
+    conversation = await manager.load()
+
+    if not conversation:
+        raise ValueError("Conversation file is empty or corrupted")
+
+    # Get agent messages only (not UI messages)
+    agent_messages = conversation.get_agent_messages()
+
+    if not agent_messages:
+        raise ValueError("No agent messages found in conversation")
+
+    # Get model config (use default provider settings)
+    model_config = await get_provider_model()
+
+    # Debug: Log the model being used
+    logger.debug(f"Using model: {model_config.name.value}")
+    logger.debug(f"Provider: {model_config.provider.value}")
+    logger.debug(f"Key provider: {model_config.key_provider.value}")
+    logger.debug(f"Max input tokens: {model_config.max_input_tokens}")
+
+    # Analyze with ContextAnalyzer
+    analyzer = ContextAnalyzer(model_config)
+    # For CLI, agent_messages and ui_message_history are the same (no hints in CLI mode)
+    analysis = await analyzer.analyze_conversation(agent_messages, list(agent_messages))
+
+    # Use formatter to generate markdown and JSON
+    markdown = ContextFormatter.format_markdown(analysis)
+    json_data = ContextFormatter.format_json(analysis)
+
+    return ContextAnalysisOutput(markdown=markdown, json_data=json_data)

shotgun/cli/export.py

@@ -63,7 +63,7 @@ def export(
         )
 
         # Create the export agent with deps and provider
-        agent, deps = create_export_agent(agent_runtime_options, provider)
+        agent, deps = asyncio.run(create_export_agent(agent_runtime_options, provider))
 
         # Start export process
         logger.info("🎯 Starting export...")

shotgun/cli/feedback.py

@@ -28,9 +28,11 @@ def send_feedback(
     ],
 ) -> None:
     """Initialize Shotgun configuration."""
+    import asyncio
+
     config_manager = get_config_manager()
-    config_manager.load()
-    shotgun_instance_id = config_manager.get_shotgun_instance_id()
+    asyncio.run(config_manager.load())
+    shotgun_instance_id = asyncio.run(config_manager.get_shotgun_instance_id())
 
     if not description:
         console.print(

shotgun/cli/models.py

@@ -8,3 +8,4 @@ class OutputFormat(StrEnum):
 
     TEXT = "text"
     JSON = "json"
+    MARKDOWN = "markdown"