shotgun-sh 0.2.11.dev5__py3-none-any.whl → 0.2.17.dev1__py3-none-any.whl

shotgun/agents/agent_manager.py

@@ -58,7 +58,12 @@ from shotgun.agents.context_analyzer import (
     ContextCompositionTelemetry,
     ContextFormatter,
 )
-from shotgun.agents.models import AgentResponse, AgentType, FileOperation
+from shotgun.agents.models import (
+    AgentResponse,
+    AgentType,
+    FileOperation,
+    FileOperationTracker,
+)
 from shotgun.posthog_telemetry import track_event
 from shotgun.tui.screens.chat_screen.hint_message import HintMessage
 from shotgun.utils.source_detection import detect_source
@@ -769,6 +774,12 @@ class AgentManager(Widget):
                     HintMessage(message=agent_response.response)
                 )
 
+            # Add file operation hints before questions (so they appear first in UI)
+            if file_operations:
+                file_hint = self._create_file_operation_hint(file_operations)
+                if file_hint:
+                    self.ui_message_history.append(HintMessage(message=file_hint))
+
             if len(agent_response.clarifying_questions) == 1:
                 # Single question - treat as non-blocking suggestion, DON'T enter Q&A mode
                 self.ui_message_history.append(
@@ -1134,6 +1145,38 @@ class AgentManager(Widget):
             )
         )
 
+    def _create_file_operation_hint(
+        self, file_operations: list[FileOperation]
+    ) -> str | None:
+        """Create a hint message for file operations.
+
+        Args:
+            file_operations: List of file operations to create a hint for
+
+        Returns:
+            Hint message string or None if no operations
+        """
+        if not file_operations:
+            return None
+
+        tracker = FileOperationTracker(operations=file_operations)
+        display_path = tracker.get_display_path()
+
+        if not display_path:
+            return None
+
+        path_obj = Path(display_path)
+
+        if len(file_operations) == 1:
+            return f"📝 Modified: `{display_path}`"
+        else:
+            num_files = len({op.file_path for op in file_operations})
+            if path_obj.is_dir():
+                return f"📁 Modified {num_files} files in: `{display_path}`"
+            else:
+                # Common path is a file, show parent directory
+                return f"📁 Modified {num_files} files in: `{path_obj.parent}`"
+
     def _post_messages_updated(
         self, file_operations: list[FileOperation] | None = None
     ) -> None:

shotgun/agents/history/history_processors.py

@@ -1,7 +1,9 @@
 """History processors for managing conversation history in Shotgun agents."""
 
+from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, Any, Protocol
 
+from anthropic import APIStatusError
 from pydantic_ai import ModelSettings
 from pydantic_ai.messages import (
     ModelMessage,
@@ -14,6 +16,7 @@ from pydantic_ai.messages import (
 from shotgun.agents.llm import shotgun_model_request
 from shotgun.agents.messages import AgentSystemPrompt, SystemStatusPrompt
 from shotgun.agents.models import AgentDeps
+from shotgun.exceptions import ContextSizeLimitExceeded
 from shotgun.logging_config import get_logger
 from shotgun.posthog_telemetry import track_event
 from shotgun.prompts import PromptLoader
@@ -51,6 +54,86 @@ logger = get_logger(__name__)
 prompt_loader = PromptLoader()
 
 
+async def _safe_token_estimation(
+    estimation_func: Callable[..., Awaitable[int]],
+    model_name: str,
+    max_tokens: int,
+    *args: Any,
+    **kwargs: Any,
+) -> int:
+    """Safely estimate tokens with proper error handling.
+
+    Wraps token estimation functions to handle failures gracefully.
+    Only RuntimeError (from token counters) is wrapped in ContextSizeLimitExceeded.
+    Other errors (network, auth) are allowed to bubble up.
+
+    Args:
+        estimation_func: Async function that estimates tokens
+        model_name: Name of the model for error messages
+        max_tokens: Maximum tokens for the model
+        *args: Arguments to pass to estimation_func
+        **kwargs: Keyword arguments to pass to estimation_func
+
+    Returns:
+        Token count from estimation_func
+
+    Raises:
+        ContextSizeLimitExceeded: If token counting fails with RuntimeError
+        Exception: Any other exceptions from estimation_func
+    """
+    try:
+        return await estimation_func(*args, **kwargs)
+    except Exception as e:
+        # Log the error with full context
+        logger.warning(
+            f"Token counting failed for {model_name}",
+            extra={
+                "error_type": type(e).__name__,
+                "error_message": str(e),
+                "model": model_name,
+            },
+        )
+
+        # Token counting behavior with oversized context (verified via testing):
+        #
+        # 1. OpenAI/tiktoken:
+        #    - Successfully counts any size (tested with 752K tokens, no error)
+        #    - Library errors: ValueError, KeyError, AttributeError, SSLError (file/cache issues)
+        #    - Wrapped as: RuntimeError by our counter
+        #
+        # 2. Gemini/SentencePiece:
+        #    - Successfully counts any size (tested with 752K tokens, no error)
+        #    - Library errors: RuntimeError, IOError, TypeError (file/model loading issues)
+        #    - Wrapped as: RuntimeError by our counter
+        #
+        # 3. Anthropic API:
+        #    - Successfully counts large token counts (tested with 752K tokens, no error)
+        #    - Only enforces 32 MB request size limit (not token count)
+        #    - Raises: APIStatusError(413) with error type 'request_too_large' for 32MB+ requests
+        #    - Other API errors: APIConnectionError, RateLimitError, APIStatusError (4xx/5xx)
+        #    - Wrapped as: RuntimeError by our counter
+        #
+        # IMPORTANT: No provider raises errors for "too many tokens" during counting.
+        # Token count validation happens separately by comparing count to max_input_tokens.
+        #
+        # We wrap RuntimeError (library-level failures from tiktoken/sentencepiece).
+        # We also wrap Anthropic's 413 error (request exceeds 32 MB) as it indicates
+        # context is effectively too large and needs user action to reduce it.
+        if isinstance(e, RuntimeError):
+            raise ContextSizeLimitExceeded(
+                model_name=model_name, max_tokens=max_tokens
+            ) from e
+
+        # Check for Anthropic's 32 MB request size limit (APIStatusError with status 413)
+        if isinstance(e, APIStatusError) and e.status_code == 413:
+            raise ContextSizeLimitExceeded(
+                model_name=model_name, max_tokens=max_tokens
+            ) from e
+
+        # Re-raise other exceptions (network errors, auth failures, etc.)
+        raise
+
+
 def is_summary_part(part: Any) -> bool:
     """Check if a message part is a compacted summary."""
     return isinstance(part, TextPart) and part.content.startswith(SUMMARY_MARKER)
@@ -157,9 +240,15 @@ async def token_limit_compactor(
 
     if last_summary_index is not None:
         # Check if post-summary conversation exceeds threshold for incremental compaction
-        post_summary_tokens = await estimate_post_summary_tokens(
-            messages, last_summary_index, deps.llm_model
+        post_summary_tokens = await _safe_token_estimation(
+            estimate_post_summary_tokens,
+            deps.llm_model.name,
+            model_max_tokens,
+            messages,
+            last_summary_index,
+            deps.llm_model,
         )
+
         post_summary_percentage = (
             (post_summary_tokens / max_tokens) * 100 if max_tokens > 0 else 0
         )
@@ -366,7 +455,14 @@ async def token_limit_compactor(
 
     else:
         # Check if total conversation exceeds threshold for full compaction
-        total_tokens = await estimate_tokens_from_messages(messages, deps.llm_model)
+        total_tokens = await _safe_token_estimation(
+            estimate_tokens_from_messages,
+            deps.llm_model.name,
+            model_max_tokens,
+            messages,
+            deps.llm_model,
+        )
+
         total_percentage = (total_tokens / max_tokens) * 100 if max_tokens > 0 else 0
 
         logger.debug(

shotgun/agents/history/token_counting/openai.py

@@ -63,7 +63,9 @@ class OpenAITokenCounter(TokenCounter):
 
         try:
             return len(self.encoding.encode(text))
-        except Exception as e:
+        except BaseException as e:
+            # Must catch BaseException to handle PanicException from tiktoken's Rust layer
+            # which can occur with extremely long texts. Regular Exception won't catch it.
             raise RuntimeError(
                 f"Failed to count tokens for OpenAI model {self.model_name}"
             ) from e

shotgun/build_constants.py

@@ -8,7 +8,7 @@ DO NOT EDIT MANUALLY.
 SENTRY_DSN = 'https://2818a6d165c64eccc94cfd51ce05d6aa@o4506813296738304.ingest.us.sentry.io/4510045952409600'
 
 # PostHog configuration embedded at build time (empty strings if not provided)
-POSTHOG_API_KEY = ''
+POSTHOG_API_KEY = 'phc_KKnChzZUKeNqZDOTJ6soCBWNQSx3vjiULdwTR9H5Mcr'
 POSTHOG_PROJECT_ID = '191396'
 
 # Logfire configuration embedded at build time (only for dev builds)

shotgun/exceptions.py

@@ -0,0 +1,32 @@
+"""General exceptions for Shotgun application."""
+
+
+class ErrorNotPickedUpBySentry(Exception):  # noqa: N818
+    """Base for user-actionable errors that shouldn't be sent to Sentry.
+
+    These errors represent expected user conditions requiring action
+    rather than bugs that need tracking.
+    """
+
+
+class ContextSizeLimitExceeded(ErrorNotPickedUpBySentry):
+    """Raised when conversation context exceeds the model's limits.
+
+    This is a user-actionable error - they need to either:
+    1. Switch to a larger context model
+    2. Switch to a larger model, compact their conversation, then switch back
+    3. Clear the conversation and start fresh
+    """
+
+    def __init__(self, model_name: str, max_tokens: int):
+        """Initialize the exception.
+
+        Args:
+            model_name: Name of the model whose limit was exceeded
+            max_tokens: Maximum tokens allowed by the model
+        """
+        self.model_name = model_name
+        self.max_tokens = max_tokens
+        super().__init__(
+            f"Context too large for {model_name} (limit: {max_tokens:,} tokens)"
+        )

shotgun/posthog_telemetry.py

@@ -18,6 +18,9 @@ logger = get_early_logger(__name__)
 # Global PostHog client instance
 _posthog_client = None
 
+# Cache the shotgun instance ID to avoid async calls during event tracking
+_shotgun_instance_id: str | None = None
+
 
 def setup_posthog_observability() -> bool:
     """Set up PostHog analytics for usage tracking.
@@ -25,7 +28,7 @@ def setup_posthog_observability() -> bool:
     Returns:
         True if PostHog was successfully set up, False otherwise
     """
-    global _posthog_client
+    global _posthog_client, _shotgun_instance_id
 
     try:
         # Check if PostHog is already initialized
@@ -57,31 +60,20 @@ def setup_posthog_observability() -> bool:
         # Store the client for later use
         _posthog_client = posthog
 
-        # Set user context with anonymous shotgun instance ID from config
+        # Cache the shotgun instance ID for later use (avoids async issues)
         try:
             import asyncio
 
             config_manager = get_config_manager()
-            shotgun_instance_id = asyncio.run(config_manager.get_shotgun_instance_id())
-
-            # Identify the user in PostHog
-            posthog.identify(  # type: ignore[attr-defined]
-                distinct_id=shotgun_instance_id,
-                properties={
-                    "version": __version__,
-                    "environment": environment,
-                },
-            )
-
-            # Set default properties for all events
-            posthog.disabled = False
-            posthog.personal_api_key = None  # Not needed for event tracking
+            _shotgun_instance_id = asyncio.run(config_manager.get_shotgun_instance_id())
 
             logger.debug(
-                "PostHog user identified with anonymous ID: %s", shotgun_instance_id
+                "PostHog initialized with shotgun instance ID: %s",
+                _shotgun_instance_id,
             )
         except Exception as e:
-            logger.warning("Failed to set user context: %s", e)
+            logger.warning("Failed to load shotgun instance ID: %s", e)
+            # Continue anyway - we'll try to get it during event tracking
 
         logger.debug(
             "PostHog analytics configured successfully (environment: %s, version: %s)",
@@ -102,18 +94,19 @@ def track_event(event_name: str, properties: dict[str, Any] | None = None) -> No
         event_name: Name of the event to track
         properties: Optional properties to include with the event
     """
-    global _posthog_client
+    global _posthog_client, _shotgun_instance_id
 
     if _posthog_client is None:
         logger.debug("PostHog not initialized, skipping event: %s", event_name)
         return
 
     try:
-        import asyncio
-
-        # Get shotgun instance ID for tracking
-        config_manager = get_config_manager()
-        shotgun_instance_id = asyncio.run(config_manager.get_shotgun_instance_id())
+        # Use cached instance ID (loaded during setup)
+        if _shotgun_instance_id is None:
+            logger.warning(
+                "Shotgun instance ID not available, skipping event: %s", event_name
+            )
+            return
 
         # Add version and environment to properties
         if properties is None:
@@ -128,7 +121,7 @@ def track_event(event_name: str, properties: dict[str, Any] | None = None) -> No
 
         # Track the event using PostHog's capture method
         _posthog_client.capture(
-            distinct_id=shotgun_instance_id, event=event_name, properties=properties
+            distinct_id=_shotgun_instance_id, event=event_name, properties=properties
         )
         logger.debug("Tracked PostHog event: %s", event_name)
     except Exception as e:

shotgun/prompts/agents/partials/common_agent_system_prompt.j2

@@ -7,10 +7,11 @@ Your extensive expertise spans, among other things:
 ## KEY RULES
 
 {% if interactive_mode %}
-0. Always ask CLARIFYING QUESTIONS using structured output if the user's request is ambiguous or lacks sufficient detail.
+0. Always ask CLARIFYING QUESTIONS using structured output before doing work.
    - Return your response with the clarifying_questions field populated
-   - Do not make assumptions about what the user wants
+   - Do not make assumptions about what the user wants, get a clear understanding first.
    - Questions should be clear, specific, and answerable
+   - Do not ask too many questions that might overwhelm the user; prioritize the most important ones.
 {% endif %}
 1. Above all, prefer using tools to do the work and NEVER respond with text.
 2. IMPORTANT: Always ask for review and go ahead to move forward after using write_file().

shotgun/sentry_telemetry.py

@@ -1,5 +1,8 @@
 """Sentry observability setup for Shotgun."""
 
+from pathlib import Path
+from typing import Any
+
 from shotgun import __version__
 from shotgun.logging_config import get_early_logger
 from shotgun.settings import settings
@@ -8,6 +11,122 @@ from shotgun.settings import settings
 logger = get_early_logger(__name__)
 
 
+def _scrub_path(path: str) -> str:
+    """Scrub sensitive information from file paths.
+
+    Removes home directory and current working directory prefixes to prevent
+    leaking usernames that might be part of the path.
+
+    Args:
+        path: The file path to scrub
+
+    Returns:
+        The scrubbed path with sensitive prefixes removed
+    """
+    if not path:
+        return path
+
+    try:
+        # Get home and cwd as Path objects for comparison
+        home = Path.home()
+        cwd = Path.cwd()
+
+        # Convert path to Path object
+        path_obj = Path(path)
+
+        # Try to make path relative to cwd first (most common case)
+        try:
+            relative_to_cwd = path_obj.relative_to(cwd)
+            return str(relative_to_cwd)
+        except ValueError:
+            pass
+
+        # Try to replace home directory with ~
+        try:
+            relative_to_home = path_obj.relative_to(home)
+            return f"~/{relative_to_home}"
+        except ValueError:
+            pass
+
+        # If path is absolute but not under cwd or home, just return filename
+        if path_obj.is_absolute():
+            return path_obj.name
+
+        # Return as-is if already relative
+        return path
+
+    except Exception:
+        # If anything goes wrong, return the original path
+        # Better to leak a path than break error reporting
+        return path
+
+
+def _scrub_sensitive_paths(event: dict[str, Any]) -> None:
+    """Scrub sensitive paths from Sentry event data.
+
+    Modifies the event in-place to remove:
+    - Home directory paths (might contain usernames)
+    - Current working directory paths (might contain usernames)
+    - Server name/hostname
+    - Paths in sys.argv
+
+    Args:
+        event: The Sentry event dictionary to scrub
+    """
+    extra = event.get("extra", {})
+    if "sys.argv" in extra:
+        argv = extra["sys.argv"]
+        if isinstance(argv, list):
+            extra["sys.argv"] = [
+                _scrub_path(arg) if isinstance(arg, str) else arg for arg in argv
+            ]
+
+    # Scrub server name if present
+    if "server_name" in event:
+        event["server_name"] = ""
+
+    # Scrub contexts that might contain paths
+    if "contexts" in event:
+        contexts = event["contexts"]
+        # Remove runtime context if it has CWD
+        if "runtime" in contexts:
+            if "cwd" in contexts["runtime"]:
+                del contexts["runtime"]["cwd"]
+            # Scrub sys.argv to remove paths
+            if "sys.argv" in contexts["runtime"]:
+                argv = contexts["runtime"]["sys.argv"]
+                if isinstance(argv, list):
+                    contexts["runtime"]["sys.argv"] = [
+                        _scrub_path(arg) if isinstance(arg, str) else arg
+                        for arg in argv
+                    ]
+
+    # Scrub exception stack traces
+    if "exception" in event and "values" in event["exception"]:
+        for exception in event["exception"]["values"]:
+            if "stacktrace" in exception and "frames" in exception["stacktrace"]:
+                for frame in exception["stacktrace"]["frames"]:
+                    # Scrub file paths
+                    if "abs_path" in frame:
+                        frame["abs_path"] = _scrub_path(frame["abs_path"])
+                    if "filename" in frame:
+                        frame["filename"] = _scrub_path(frame["filename"])
+
+                    # Scrub local variables that might contain paths
+                    if "vars" in frame:
+                        for var_name, var_value in frame["vars"].items():
+                            if isinstance(var_value, str):
+                                frame["vars"][var_name] = _scrub_path(var_value)
+
+    # Scrub breadcrumbs that might contain paths
+    if "breadcrumbs" in event and "values" in event["breadcrumbs"]:
+        for breadcrumb in event["breadcrumbs"]["values"]:
+            if "data" in breadcrumb:
+                for key, value in breadcrumb["data"].items():
+                    if isinstance(value, str):
+                        breadcrumb["data"][key] = _scrub_path(value)
+
+
 def setup_sentry_observability() -> bool:
     """Set up Sentry observability for error tracking.
 
@@ -32,20 +151,57 @@ def setup_sentry_observability() -> bool:
         logger.debug("Using Sentry DSN from settings, proceeding with setup")
 
         # Determine environment based on version
-        # Dev versions contain "dev", "rc", "alpha", or "beta"
+        # Dev versions contain "dev", "rc", "alpha", "beta"
         if any(marker in __version__ for marker in ["dev", "rc", "alpha", "beta"]):
             environment = "development"
         else:
             environment = "production"
 
+        def before_send(event: Any, hint: dict[str, Any]) -> Any:
+            """Filter out user-actionable errors and scrub sensitive paths.
+
+            User-actionable errors (like context size limits) are expected conditions
+            that users need to resolve, not bugs that need tracking.
+
+            Also scrubs sensitive information like usernames from file paths and
+            working directories to protect user privacy.
+            """
+
+            log_record = hint.get("log_record")
+            if log_record:
+                # Scrub pathname using the helper function
+                log_record.pathname = _scrub_path(log_record.pathname)
+
+                # Scrub traceback text if it exists
+                if hasattr(log_record, "exc_text") and isinstance(
+                    log_record.exc_text, str
+                ):
+                    # Replace home directory in traceback text
+                    home = Path.home()
+                    log_record.exc_text = log_record.exc_text.replace(str(home), "~")
+
+            if "exc_info" in hint:
+                _, exc_value, _ = hint["exc_info"]
+                from shotgun.exceptions import ErrorNotPickedUpBySentry
+
+                if isinstance(exc_value, ErrorNotPickedUpBySentry):
+                    # Don't send to Sentry - this is user-actionable, not a bug
+                    return None
+
+            # Scrub sensitive paths from the event
+            _scrub_sensitive_paths(event)
+            return event
+
         # Initialize Sentry
         sentry_sdk.init(
             dsn=dsn,
             release=f"shotgun-sh@{__version__}",
             environment=environment,
             send_default_pii=False,  # Privacy-first: never send PII
+            server_name="",  # Privacy: don't send hostname (may contain username)
             traces_sample_rate=0.1 if environment == "production" else 1.0,
             profiles_sample_rate=0.1 if environment == "production" else 1.0,
+            before_send=before_send,
         )
 
         # Set user context with anonymous shotgun instance ID from config