shotgun-sh 0.2.3.dev2__py3-none-any.whl → 0.2.11.dev1__py3-none-any.whl

shotgun/cli/compact.py

@@ -0,0 +1,186 @@
+"""Compact command for shotgun CLI."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from pydantic_ai.usage import RequestUsage
+from rich.console import Console
+
+from shotgun.agents.config import get_provider_model
+from shotgun.agents.conversation_manager import ConversationManager
+from shotgun.agents.history.history_processors import token_limit_compactor
+from shotgun.agents.history.token_estimation import estimate_tokens_from_messages
+from shotgun.cli.models import OutputFormat
+from shotgun.logging_config import get_logger
+
+app = typer.Typer(
+    name="compact", help="Compact the conversation history", no_args_is_help=False
+)
+logger = get_logger(__name__)
+console = Console()
+
+
+@app.callback(invoke_without_command=True)
+def compact(
+    format: Annotated[
+        OutputFormat,
+        typer.Option(
+            "--format",
+            "-f",
+            help="Output format: markdown or json",
+        ),
+    ] = OutputFormat.MARKDOWN,
+) -> None:
+    """Compact the current conversation history to reduce size.
+
+    This command compacts the conversation in ~/.shotgun-sh/conversation.json
+    by summarizing older messages while preserving recent context. The compacted
+    conversation is automatically saved back to the file.
+    """
+    try:
+        result = asyncio.run(compact_conversation())
+
+        if format == OutputFormat.JSON:
+            # Output as JSON
+            console.print_json(json.dumps(result, indent=2))
+        else:
+            # Output as markdown
+            console.print(format_markdown(result))
+
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error:[/red] {e}\n\n"
+            "No conversation found. Start a TUI session first with: [cyan]shotgun[/cyan]",
+            style="bold",
+        )
+        raise typer.Exit(code=1) from e
+    except Exception as e:
+        console.print(
+            f"[red]Error:[/red] Failed to compact conversation: {e}", style="bold"
+        )
+        logger.debug("Full traceback:", exc_info=True)
+        raise typer.Exit(code=1) from e
+
+
+async def compact_conversation() -> dict[str, Any]:
+    """Compact the conversation and return statistics.
+
+    Returns:
+        Dictionary with compaction statistics including before/after metrics
+    """
+    # Get conversation file path
+    conversation_file = Path.home() / ".shotgun-sh" / "conversation.json"
+
+    if not conversation_file.exists():
+        raise FileNotFoundError(f"Conversation file not found at {conversation_file}")
+
+    # Load conversation
+    manager = ConversationManager(conversation_file)
+    conversation = manager.load()
+
+    if not conversation:
+        raise ValueError("Conversation file is empty or corrupted")
+
+    # Get agent messages only (not UI messages)
+    agent_messages = conversation.get_agent_messages()
+
+    if not agent_messages:
+        raise ValueError("No agent messages found in conversation")
+
+    # Get model config
+    model_config = get_provider_model()
+
+    # Calculate before metrics
+    original_message_count = len(agent_messages)
+    original_tokens = await estimate_tokens_from_messages(agent_messages, model_config)
+
+    # For CLI, we can call token_limit_compactor directly without full AgentDeps
+    # since we only need the model config and message history
+    # Create a minimal context object for compaction
+    class CompactContext:
+        def __init__(self, model_config: Any, usage: RequestUsage) -> None:
+            self.deps = type("Deps", (), {"llm_model": model_config})()
+            self.usage = usage
+
+    # Create minimal usage info for compaction check
+    usage = RequestUsage(input_tokens=original_tokens, output_tokens=0)
+    ctx = CompactContext(model_config, usage)
+
+    # Apply compaction with force=True to bypass threshold checks
+    compacted_messages = await token_limit_compactor(ctx, agent_messages, force=True)
+
+    # Calculate after metrics
+    compacted_message_count = len(compacted_messages)
+    compacted_tokens = await estimate_tokens_from_messages(
+        compacted_messages, model_config
+    )
+
+    # Calculate reduction percentages
+    message_reduction = (
+        ((original_message_count - compacted_message_count) / original_message_count)
+        * 100
+        if original_message_count > 0
+        else 0
+    )
+    token_reduction = (
+        ((original_tokens - compacted_tokens) / original_tokens) * 100
+        if original_tokens > 0
+        else 0
+    )
+
+    # Save compacted conversation
+    conversation.set_agent_messages(compacted_messages)
+    manager.save(conversation)
+
+    logger.info(
+        f"Compacted conversation: {original_message_count} → {compacted_message_count} messages "
+        f"({message_reduction:.1f}% reduction)"
+    )
+
+    return {
+        "success": True,
+        "before": {
+            "messages": original_message_count,
+            "estimated_tokens": original_tokens,
+        },
+        "after": {
+            "messages": compacted_message_count,
+            "estimated_tokens": compacted_tokens,
+        },
+        "reduction": {
+            "messages_percent": round(message_reduction, 1),
+            "tokens_percent": round(token_reduction, 1),
+        },
+    }
+
+
+def format_markdown(result: dict[str, Any]) -> str:
+    """Format compaction result as markdown.
+
+    Args:
+        result: Dictionary with compaction statistics
+
+    Returns:
+        Formatted markdown string
+    """
+    before = result["before"]
+    after = result["after"]
+    reduction = result["reduction"]
+
+    return f"""# Conversation Compacted ✓
+
+## Before
+- **Messages:** {before["messages"]:,}
+- **Estimated Tokens:** {before["estimated_tokens"]:,}
+
+## After
+- **Messages:** {after["messages"]:,}
+- **Estimated Tokens:** {after["estimated_tokens"]:,}
+
+## Reduction
+- **Messages:** {reduction["messages_percent"]}%
+- **Tokens:** {reduction["tokens_percent"]}%
+"""

shotgun/cli/context.py

@@ -0,0 +1,111 @@
+"""Context command for shotgun CLI."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from shotgun.agents.config import get_provider_model
+from shotgun.agents.context_analyzer import (
+    ContextAnalysisOutput,
+    ContextAnalyzer,
+    ContextFormatter,
+)
+from shotgun.agents.conversation_manager import ConversationManager
+from shotgun.cli.models import OutputFormat
+from shotgun.logging_config import get_logger
+
+app = typer.Typer(
+    name="context", help="Analyze conversation context usage", no_args_is_help=False
+)
+logger = get_logger(__name__)
+console = Console()
+
+
+@app.callback(invoke_without_command=True)
+def context(
+    format: Annotated[
+        OutputFormat,
+        typer.Option(
+            "--format",
+            "-f",
+            help="Output format: markdown or json",
+        ),
+    ] = OutputFormat.MARKDOWN,
+) -> None:
+    """Analyze the current conversation's context usage.
+
+    This command analyzes the agent's message history from ~/.shotgun-sh/conversation.json
+    and displays token usage breakdown by message type. Only agent context is counted
+    (UI elements like hints are excluded).
+    """
+    try:
+        result = asyncio.run(analyze_context())
+
+        if format == OutputFormat.JSON:
+            # Output as JSON
+            console.print_json(json.dumps(result.json_data, indent=2))
+        else:
+            # Output as plain text (Markdown() reformats and makes categories inline)
+            console.print(result.markdown)
+
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error:[/red] {e}\n\n"
+            "No conversation found. Start a TUI session first with: [cyan]shotgun[/cyan]",
+            style="bold",
+        )
+        raise typer.Exit(code=1) from e
+    except Exception as e:
+        console.print(f"[red]Error:[/red] Failed to analyze context: {e}", style="bold")
+        logger.debug("Full traceback:", exc_info=True)
+        raise typer.Exit(code=1) from e
+
+
+async def analyze_context() -> ContextAnalysisOutput:
+    """Analyze the conversation context and return structured data.
+
+    Returns:
+        ContextAnalysisOutput with both markdown and JSON representations of the analysis
+    """
+    # Get conversation file path
+    conversation_file = Path.home() / ".shotgun-sh" / "conversation.json"
+
+    if not conversation_file.exists():
+        raise FileNotFoundError(f"Conversation file not found at {conversation_file}")
+
+    # Load conversation
+    manager = ConversationManager(conversation_file)
+    conversation = manager.load()
+
+    if not conversation:
+        raise ValueError("Conversation file is empty or corrupted")
+
+    # Get agent messages only (not UI messages)
+    agent_messages = conversation.get_agent_messages()
+
+    if not agent_messages:
+        raise ValueError("No agent messages found in conversation")
+
+    # Get model config (use default provider settings)
+    model_config = get_provider_model()
+
+    # Debug: Log the model being used
+    logger.debug(f"Using model: {model_config.name.value}")
+    logger.debug(f"Provider: {model_config.provider.value}")
+    logger.debug(f"Key provider: {model_config.key_provider.value}")
+    logger.debug(f"Max input tokens: {model_config.max_input_tokens}")
+
+    # Analyze with ContextAnalyzer
+    analyzer = ContextAnalyzer(model_config)
+    # For CLI, agent_messages and ui_message_history are the same (no hints in CLI mode)
+    analysis = await analyzer.analyze_conversation(agent_messages, list(agent_messages))
+
+    # Use formatter to generate markdown and JSON
+    markdown = ContextFormatter.format_markdown(analysis)
+    json_data = ContextFormatter.format_json(analysis)
+
+    return ContextAnalysisOutput(markdown=markdown, json_data=json_data)

shotgun/cli/models.py

@@ -8,3 +8,4 @@ class OutputFormat(StrEnum):
 
     TEXT = "text"
     JSON = "json"
+    MARKDOWN = "markdown"

shotgun/cli/update.py

@@ -45,7 +45,7 @@ def update(
 
     This command will:
     - Check PyPI for the latest version
-    - Detect your installation method (pipx, pip, or venv)
+    - Detect your installation method (uvx, uv-tool, pipx, pip, or venv)
     - Perform the appropriate upgrade command
 
     Examples:
@@ -93,6 +93,8 @@ def update(
         )
         console.print(
             "Use --force to update anyway, or install the stable version with:\n"
+            "  uv tool install shotgun-sh\n"
+            "  or\n"
             "  pipx install shotgun-sh\n"
             "  or\n"
             "  pip install shotgun-sh",
@@ -134,7 +136,19 @@ def update(
             console.print(f"\n[red]✗[/red] {message}", style="bold red")
 
             # Provide manual update instructions
-            if method == "pipx":
+            if method == "uvx":
+                console.print(
+                    "\n[yellow]Run uvx again to use the latest version:[/yellow]\n"
+                    "  uvx shotgun-sh\n"
+                    "\n[yellow]Or install permanently:[/yellow]\n"
+                    "  uv tool install shotgun-sh"
+                )
+            elif method == "uv-tool":
+                console.print(
+                    "\n[yellow]Try updating manually:[/yellow]\n"
+                    "  uv tool upgrade shotgun-sh"
+                )
+            elif method == "pipx":
                 console.print(
                     "\n[yellow]Try updating manually:[/yellow]\n"
                     "  pipx upgrade shotgun-sh"

shotgun/codebase/core/manager.py

@@ -371,7 +371,16 @@ class CodebaseGraphManager:
                 )
                 import shutil
 
-                shutil.rmtree(graph_path)
+                # Handle both files and directories (kuzu v0.11.2+ uses files)
+                if graph_path.is_file():
+                    graph_path.unlink()  # Delete file
+                    # Also delete WAL file if it exists
+                    wal_path = graph_path.with_suffix(graph_path.suffix + ".wal")
+                    if wal_path.exists():
+                        wal_path.unlink()
+                        logger.debug(f"Deleted WAL file: {wal_path}")
+                else:
+                    shutil.rmtree(graph_path)  # Delete directory
 
         # Import the builder from local core module
         from shotgun.codebase.core import CodebaseIngestor

shotgun/llm_proxy/__init__.py

@@ -1,6 +1,9 @@
 """LiteLLM proxy client utilities and configuration."""
 
-from .clients import create_anthropic_proxy_client, create_litellm_provider
+from .clients import (
+    create_anthropic_proxy_provider,
+    create_litellm_provider,
+)
 from .constants import (
     LITELLM_PROXY_ANTHROPIC_BASE,
     LITELLM_PROXY_BASE_URL,
@@ -12,5 +15,5 @@ __all__ = [
     "LITELLM_PROXY_ANTHROPIC_BASE",
     "LITELLM_PROXY_OPENAI_BASE",
     "create_litellm_provider",
-    "create_anthropic_proxy_client",
+    "create_anthropic_proxy_provider",
 ]

shotgun/llm_proxy/clients.py

@@ -1,6 +1,6 @@
 """Client creation utilities for LiteLLM proxy."""
 
-from anthropic import Anthropic
+from pydantic_ai.providers.anthropic import AnthropicProvider
 from pydantic_ai.providers.litellm import LiteLLMProvider
 
 from .constants import LITELLM_PROXY_ANTHROPIC_BASE, LITELLM_PROXY_BASE_URL
@@ -21,19 +21,24 @@ def create_litellm_provider(api_key: str) -> LiteLLMProvider:
     )
 
 
-def create_anthropic_proxy_client(api_key: str) -> Anthropic:
-    """Create Anthropic client configured for LiteLLM proxy.
+def create_anthropic_proxy_provider(api_key: str) -> AnthropicProvider:
+    """Create Anthropic provider configured for LiteLLM proxy.
 
-    This client will proxy token counting requests through the
-    LiteLLM proxy to Anthropic's actual token counting API.
+    This provider uses native Anthropic API format while routing through
+    the LiteLLM proxy. This preserves Anthropic-specific features like
+    tool_choice and web search.
+
+    The provider's .client attribute provides access to the async Anthropic
+    client (AsyncAnthropic), which should be used for all API operations
+    including token counting.
 
     Args:
         api_key: Shotgun API key
 
     Returns:
-        Anthropic client configured to use LiteLLM proxy
+        AnthropicProvider configured to use LiteLLM proxy /anthropic endpoint
     """
-    return Anthropic(
+    return AnthropicProvider(
         api_key=api_key,
         base_url=LITELLM_PROXY_ANTHROPIC_BASE,
     )

shotgun/logging_config.py

@@ -2,10 +2,10 @@
 
 import logging
 import logging.handlers
-import os
 import sys
 from pathlib import Path
 
+from shotgun.settings import settings
 from shotgun.utils.env_utils import is_truthy
 
 
@@ -75,12 +75,10 @@ def setup_logger(
     if has_file_handler:
         return logger
 
-    # Get log level from environment variable, default to INFO
-    env_level = os.getenv("SHOTGUN_LOG_LEVEL", "INFO").upper()
-    if env_level not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
-        env_level = "INFO"
+    # Get log level from settings (already validated and uppercased)
+    log_level = settings.logging.log_level
 
-    logger.setLevel(getattr(logging, env_level))
+    logger.setLevel(getattr(logging, log_level))
 
     # Default format string
     if format_string is None:
@@ -102,13 +100,13 @@ def setup_logger(
     # Check if console logging is enabled (default: off)
     # Force console logging OFF if Logfire is enabled in dev build
     console_logging_enabled = (
-        is_truthy(os.getenv("LOGGING_TO_CONSOLE", "false")) and not is_logfire_dev_build
+        settings.logging.logging_to_console and not is_logfire_dev_build
     )
 
     if console_logging_enabled:
         # Create console handler
         console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setLevel(getattr(logging, env_level))
+        console_handler.setLevel(getattr(logging, log_level))
 
         # Use colored formatter for console
         console_formatter = ColoredFormatter(format_string, datefmt="%H:%M:%S")
@@ -118,7 +116,7 @@ def setup_logger(
         logger.addHandler(console_handler)
 
     # Check if file logging is enabled (default: on)
-    file_logging_enabled = is_truthy(os.getenv("LOGGING_TO_FILE", "true"))
+    file_logging_enabled = settings.logging.logging_to_file
 
     if file_logging_enabled:
         try:
@@ -137,7 +135,7 @@ def setup_logger(
 
             # Also set max file size (10MB) using RotatingFileHandler as fallback
             # Note: We'll use TimedRotatingFileHandler which handles both time and size
-            file_handler.setLevel(getattr(logging, env_level))
+            file_handler.setLevel(getattr(logging, log_level))
 
             # Use standard formatter for file (no colors)
             file_formatter = logging.Formatter(

shotgun/main.py

@@ -23,8 +23,11 @@ from dotenv import load_dotenv
 from shotgun import __version__
 from shotgun.agents.config import get_config_manager
 from shotgun.cli import (
+    clear,
     codebase,
+    compact,
     config,
+    context,
     export,
     feedback,
     plan,
@@ -78,6 +81,9 @@ app.add_typer(config.app, name="config", help="Manage Shotgun configuration")
 app.add_typer(
     codebase.app, name="codebase", help="Manage and query code knowledge graphs"
 )
+app.add_typer(context.app, name="context", help="Analyze conversation context usage")
+app.add_typer(compact.app, name="compact", help="Compact conversation history")
+app.add_typer(clear.app, name="clear", help="Clear conversation history")
 app.add_typer(research.app, name="research", help="Perform research with agentic loops")
 app.add_typer(plan.app, name="plan", help="Generate structured plans")
 app.add_typer(specify.app, name="specify", help="Generate comprehensive specifications")
@@ -125,6 +131,41 @@ def main(
             help="Continue previous TUI conversation",
         ),
     ] = False,
+    web: Annotated[
+        bool,
+        typer.Option(
+            "--web",
+            help="Serve TUI as web application",
+        ),
+    ] = False,
+    port: Annotated[
+        int,
+        typer.Option(
+            "--port",
+            help="Port for web server (only used with --web)",
+        ),
+    ] = 8000,
+    host: Annotated[
+        str,
+        typer.Option(
+            "--host",
+            help="Host address for web server (only used with --web)",
+        ),
+    ] = "localhost",
+    public_url: Annotated[
+        str | None,
+        typer.Option(
+            "--public-url",
+            help="Public URL if behind proxy (only used with --web)",
+        ),
+    ] = None,
+    force_reindex: Annotated[
+        bool,
+        typer.Option(
+            "--force-reindex",
+            help="Force re-indexing of codebase (ignores existing index)",
+        ),
+    ] = False,
 ) -> None:
     """Shotgun - AI-powered CLI tool."""
     logger.debug("Starting shotgun CLI application")
@@ -134,16 +175,35 @@ def main(
         perform_auto_update_async(no_update_check=no_update_check)
 
     if ctx.invoked_subcommand is None and not ctx.resilient_parsing:
-        logger.debug("Launching shotgun TUI application")
-        try:
-            tui_app.run(
-                no_update_check=no_update_check, continue_session=continue_session
-            )
-        finally:
-            # Ensure PostHog is shut down cleanly even if TUI exits unexpectedly
-            from shotgun.posthog_telemetry import shutdown
-
-            shutdown()
+        if web:
+            logger.debug("Launching shotgun TUI as web application")
+            try:
+                tui_app.serve(
+                    host=host,
+                    port=port,
+                    public_url=public_url,
+                    no_update_check=no_update_check,
+                    continue_session=continue_session,
+                    force_reindex=force_reindex,
+                )
+            finally:
+                # Ensure PostHog is shut down cleanly even if server exits unexpectedly
+                from shotgun.posthog_telemetry import shutdown
+
+                shutdown()
+        else:
+            logger.debug("Launching shotgun TUI application")
+            try:
+                tui_app.run(
+                    no_update_check=no_update_check,
+                    continue_session=continue_session,
+                    force_reindex=force_reindex,
+                )
+            finally:
+                # Ensure PostHog is shut down cleanly even if TUI exits unexpectedly
+                from shotgun.posthog_telemetry import shutdown
+
+                shutdown()
         raise typer.Exit()
 
     # For CLI commands, register PostHog shutdown handler

shotgun/posthog_telemetry.py

@@ -10,6 +10,7 @@ from shotgun import __version__
 from shotgun.agents.config import get_config_manager
 from shotgun.agents.conversation_manager import ConversationManager
 from shotgun.logging_config import get_early_logger
+from shotgun.settings import settings
 
 # Use early logger to prevent automatic StreamHandler creation
 logger = get_early_logger(__name__)
@@ -32,10 +33,15 @@ def setup_posthog_observability() -> bool:
             logger.debug("PostHog is already initialized, skipping")
             return True
 
-        # Hardcoded PostHog configuration
-        api_key = "phc_KKnChzZUKeNqZDOTJ6soCBWNQSx3vjiULdwTR9H5Mcr"
+        # Get API key from settings (handles build constants + env vars automatically)
+        api_key = settings.telemetry.posthog_api_key
 
-        logger.debug("Using hardcoded PostHog configuration")
+        # If no API key is available, skip PostHog initialization
+        if not api_key:
+            logger.debug("No PostHog API key available, skipping initialization")
+            return False
+
+        logger.debug("Using PostHog API key from settings")
 
         # Determine environment based on version
         # Dev versions contain "dev", "rc", "alpha", or "beta"

shotgun/prompts/agents/export.j2

@@ -124,6 +124,7 @@ content_tasks = read_file('tasks.md')         # Read implementation details
    - `plan.md` - Extract development approach and stages
    - `tasks.md` - Understand implementation tasks and structure
 2. **Map content to agents.md standard sections**:
+   - **Research, Specifications, and Planning**: ALWAYS include this section first. Check which pipeline files exist in `.shotgun/` (research.md, specification.md, plan.md, tasks.md) and list only the ones that exist. If none exist, omit this section.
    - **Project Overview**: Brief description and key technologies from specification.md
    - **Dev Environment Setup**: Installation, dependencies, dev server commands
    - **Code Style Guidelines**: Coding conventions and patterns from research.md
@@ -170,6 +171,14 @@ For additional specialized exports (only if specifically requested):
 <CORRECT_CONTENT_TEMPLATE>
 # Agents.md - [Project Name]
 
+## Research, Specifications, and Planning
+
+The `.shotgun/` folder contains background research, specifications, and implementation planning files. Refer to these files for additional context:
+- `research.md` - Codebase analysis and research findings
+- `specification.md` - Project requirements and specifications
+- `plan.md` - Development plan and implementation approach
+- `tasks.md` - Task breakdown and implementation progress
+
 ## Project Overview
 - Brief description of what the project does
 - Key technologies and frameworks used
@@ -253,6 +262,14 @@ This project is about [making assumptions without reading files]...
 <GOOD_CONTENT_EXAMPLE>
 # Agents.md - E-commerce API Project
 
+## Research, Specifications, and Planning
+
+The `.shotgun/` folder contains background research, specifications, and implementation planning files. Refer to these files for additional context:
+- `research.md` - Codebase analysis and research findings
+- `specification.md` - Project requirements and specifications
+- `plan.md` - Development plan and implementation approach
+- `tasks.md` - Task breakdown and implementation progress
+
 ## Project Overview
 - REST API for product catalog management with authentication
 - Built with Python/FastAPI for high performance async operations
@@ -316,7 +333,7 @@ This project is about [making assumptions without reading files]...
 USER INTERACTION - CLARIFY EXPORT REQUIREMENTS:
 
 - ALWAYS ask clarifying questions when export requirements are unclear
-- Use ask_user tool to gather specific details about:
+- Use clarifying questions to gather specific details about:
   - Target format and file type preferences
   - Intended use case and audience for the export
   - Specific content sections to include/exclude from files

shotgun/prompts/agents/partials/common_agent_system_prompt.j2

@@ -7,7 +7,10 @@ Your extensive expertise spans, among other things:
 ## KEY RULES
 
 {% if interactive_mode %}
-0. Always ask CLARIFYING QUESTIONS if the user's request is ambiguous or lacks sufficient detail. Do not make assumptions about what the user wants.
+0. Always ask CLARIFYING QUESTIONS using structured output if the user's request is ambiguous or lacks sufficient detail.
+   - Return your response with the clarifying_questions field populated
+   - Do not make assumptions about what the user wants
+   - Questions should be clear, specific, and answerable
 {% endif %}
 1. Above all, prefer using tools to do the work and NEVER respond with text.
 2. IMPORTANT: Always ask for review and go ahead to move forward after using write_file().
@@ -20,6 +23,7 @@ Your extensive expertise spans, among other things:
 9. **Be Creative**: If the user seems not to know something, always be creative and come up with ideas that fit their thinking.
 10. Greet the user when you're just starting to work.
 11. DO NOT repeat yourself.
+12. If a user has agreed to a plan, you DO NOT NEED TO FOLLOW UP with them after every step to ask "is this search query ok?".
 
 
 ## Quality Standards