shotgun-sh 0.2.8.dev2__py3-none-any.whl → 0.3.3.dev1__py3-none-any.whl

shotgun/agents/{history → conversation/history}/compaction.py

@@ -13,24 +13,51 @@ logger = get_logger(__name__)
 
 
 async def apply_persistent_compaction(
-    messages: list[ModelMessage], deps: AgentDeps
+    messages: list[ModelMessage], deps: AgentDeps, force: bool = False
 ) -> list[ModelMessage]:
     """Apply compaction to message history for persistent storage.
 
     This ensures that compacted history is actually used as the conversation baseline,
     preventing cascading compaction issues across both CLI and TUI usage patterns.
 
+    Compaction happens in two phases:
+    1. Deterministic pre-compaction: Remove file content (no LLM needed)
+    2. LLM-based compaction: Summarize conversation if still over threshold
+
     Args:
         messages: Full message history from agent run
         deps: Agent dependencies containing model config
+        force: If True, force compaction even if below token threshold
 
     Returns:
         Compacted message history that should be stored as conversation state
     """
+    from .file_content_deduplication import deduplicate_file_content
     from .history_processors import token_limit_compactor
 
     try:
-        # Count actual token usage using shared utility
+        # STEP 1: Deterministic pre-compaction (no LLM cost)
+        # Remove file content from tool returns - files are still accessible
+        # via retrieve_code (codebase) or read_file (.shotgun/ folder)
+        messages, tokens_saved = deduplicate_file_content(
+            messages,
+            retention_window=3,  # Keep last 3 messages' file content intact
+        )
+
+        if tokens_saved > 0:
+            logger.info(
+                f"Pre-compaction: removed ~{tokens_saved:,} tokens of file content"
+            )
+            track_event(
+                "file_content_deduplication",
+                {
+                    "tokens_saved_estimate": tokens_saved,
+                    "retention_window": 3,
+                    "model_name": deps.llm_model.name.value,
+                },
+            )
+
+        # STEP 2: Count tokens after pre-compaction
         estimated_tokens = await estimate_tokens_from_messages(messages, deps.llm_model)
 
         # Create minimal usage info for compaction check
@@ -46,7 +73,7 @@ async def apply_persistent_compaction(
                 self.usage = usage
 
         ctx = MockContext(deps, usage)
-        compacted_messages = await token_limit_compactor(ctx, messages)
+        compacted_messages = await token_limit_compactor(ctx, messages, force=force)
 
         # Log the result for monitoring
         original_size = len(messages)
@@ -59,17 +86,21 @@ async def apply_persistent_compaction(
                 f"({reduction_pct:.1f}% reduction)"
             )
 
-            # Track persistent compaction event
+            # Track persistent compaction event with simple metrics (fast, no token counting)
             track_event(
                 "persistent_compaction_applied",
                 {
+                    # Basic compaction metrics
                     "messages_before": original_size,
                     "messages_after": compacted_size,
-                    "tokens_before": estimated_tokens,
                     "reduction_percentage": round(reduction_pct, 2),
                     "agent_mode": deps.agent_mode.value
                     if hasattr(deps, "agent_mode") and deps.agent_mode
                     else "unknown",
+                    # Model and provider info (no computation needed)
+                    "model_name": deps.llm_model.name.value,
+                    "provider": deps.llm_model.provider.value,
+                    "key_provider": deps.llm_model.key_provider.value,
                 },
             )
         else:

shotgun/agents/{history → conversation/history}/constants.py

@@ -10,6 +10,11 @@ INPUT_BUFFER_TOKENS = 500
 MIN_SUMMARY_TOKENS = 100
 TOKEN_LIMIT_RATIO = 0.8
 
+# Chunked compaction constants
+CHUNK_TARGET_RATIO = 0.60  # Target chunk size as % of max_input_tokens
+CHUNK_SAFE_RATIO = 0.70  # Max safe ratio before triggering chunked compaction
+RETENTION_WINDOW_MESSAGES = 5  # Keep last N message groups outside compaction
+
 
 class SummaryType(Enum):
     """Types of summarization requests for logging."""

shotgun/agents/conversation/history/file_content_deduplication.py

@@ -0,0 +1,216 @@
+"""Pre-compaction file content deduplication for conversation history.
+
+This module provides a deterministic pre-pass that removes file content from
+tool returns before LLM-based compaction. Files are still accessible via
+`retrieve_code` (codebase) or `read_file` (.shotgun/ folder).
+"""
+
+import copy
+import re
+from enum import StrEnum
+from typing import Any
+
+from pydantic_ai.messages import (
+    ModelMessage,
+    ModelRequest,
+    ToolReturnPart,
+)
+
+from shotgun.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class FileReadTool(StrEnum):
+    """Tool names that read file content."""
+
+    CODEBASE = "file_read"  # Reads from indexed codebase (Kuzu graph)
+    SHOTGUN_FOLDER = "read_file"  # Reads from .shotgun/ folder
+
+
+# Minimum content length to bother deduplicating (skip tiny files)
+MIN_CONTENT_LENGTH = 500
+
+# Placeholder templates for each type
+CODEBASE_PLACEHOLDER = (
+    "**File**: `{file_path}`\n"
+    "**Size**: {size_bytes} bytes | **Language**: {language}\n"
+    "**Content**: [Removed for compaction - use `retrieve_code` or `file_read` to access]"
+)
+
+SHOTGUN_PLACEHOLDER = (
+    "**File**: `.shotgun/{filename}`\n"
+    "**Content**: [Removed for compaction - file persisted in .shotgun/ folder]"
+)
+
+# Pattern for parsing file_read output (codebase files)
+# Format: **File**: `path`\n**Size**: N bytes\n[optional encoding]\n\n**Content**:\n```lang\ncontent```
+CODEBASE_FILE_PATTERN = re.compile(
+    r"\*\*File\*\*:\s*`([^`]+)`\s*\n"  # File path
+    r"\*\*Size\*\*:\s*(\d+)\s*bytes\s*\n"  # Size in bytes
+    r"(?:\*\*Encoding\*\*:.*?\n)?"  # Optional encoding line
+    r"\n\*\*Content\*\*:\s*\n"  # Blank line + Content header
+    r"```(\w*)\n"  # Language tag
+    r"(.*?)```",  # Actual content
+    re.DOTALL,
+)
+
+
+def _parse_codebase_file_content(
+    content: str,
+) -> tuple[str, int, str, str] | None:
+    """Parse file_read tool return content.
+
+    Args:
+        content: The tool return content string
+
+    Returns:
+        Tuple of (file_path, size_bytes, language, actual_content) or None if not parseable
+    """
+    match = CODEBASE_FILE_PATTERN.search(content)
+    if not match:
+        return None
+
+    file_path = match.group(1)
+    size_bytes = int(match.group(2))
+    language = match.group(3) or ""
+    actual_content = match.group(4)
+
+    return file_path, size_bytes, language, actual_content
+
+
+def _create_codebase_placeholder(file_path: str, size_bytes: int, language: str) -> str:
+    """Create placeholder for codebase file content."""
+    return CODEBASE_PLACEHOLDER.format(
+        file_path=file_path,
+        size_bytes=size_bytes,
+        language=language or "unknown",
+    )
+
+
+def _create_shotgun_placeholder(filename: str) -> str:
+    """Create placeholder for .shotgun/ file content."""
+    return SHOTGUN_PLACEHOLDER.format(filename=filename)
+
+
+def _estimate_tokens_saved(original: str, replacement: str) -> int:
+    """Rough estimate of tokens saved (~4 chars per token)."""
+    original_chars = len(original)
+    replacement_chars = len(replacement)
+    # Rough token estimate: ~4 characters per token for code
+    return max(0, (original_chars - replacement_chars) // 4)
+
+
+def deduplicate_file_content(
+    messages: list[ModelMessage],
+    retention_window: int = 3,
+) -> tuple[list[ModelMessage], int]:
+    """Replace file read content with placeholders for indexed/persisted files.
+
+    This is a deterministic pre-compaction pass that reduces tokens without
+    requiring an LLM. Files remain accessible via their respective tools.
+
+    Args:
+        messages: Conversation history
+        retention_window: Keep full content in last N messages (for recent context)
+
+    Returns:
+        Tuple of (modified_messages, estimated_tokens_saved)
+    """
+    if not messages:
+        return messages, 0
+
+    # Deep copy to avoid modifying original
+    modified_messages = copy.deepcopy(messages)
+    total_tokens_saved = 0
+    files_deduplicated = 0
+
+    # Calculate retention boundary (keep last N messages intact)
+    retention_start = max(0, len(modified_messages) - retention_window)
+
+    for msg_idx, message in enumerate(modified_messages):
+        # Skip messages in retention window
+        if msg_idx >= retention_start:
+            continue
+
+        # Only process ModelRequest (which contains ToolReturnPart)
+        if not isinstance(message, ModelRequest):
+            continue
+
+        # Build new parts list, replacing file content where appropriate
+        new_parts: list[Any] = []
+        message_modified = False
+
+        for part in message.parts:
+            if not isinstance(part, ToolReturnPart):
+                new_parts.append(part)
+                continue
+
+            tool_name = part.tool_name
+            content = part.content
+
+            # Skip if content is too short to bother
+            if not isinstance(content, str) or len(content) < MIN_CONTENT_LENGTH:
+                new_parts.append(part)
+                continue
+
+            replacement = None
+            original_content = content
+
+            # Handle codebase file reads (file_read)
+            if tool_name == FileReadTool.CODEBASE:
+                parsed = _parse_codebase_file_content(content)
+                if parsed:
+                    file_path, size_bytes, language, actual_content = parsed
+                    # Only replace if actual content is substantial
+                    if len(actual_content) >= MIN_CONTENT_LENGTH:
+                        replacement = _create_codebase_placeholder(
+                            file_path, size_bytes, language
+                        )
+                        logger.debug(
+                            f"Deduplicating codebase file: {file_path} "
+                            f"({size_bytes} bytes)"
+                        )
+
+            # Handle .shotgun/ file reads (read_file)
+            elif tool_name == FileReadTool.SHOTGUN_FOLDER:
+                # For read_file, content is raw - we need to figure out filename
+                # from the tool call args (but we only have the return here)
+                # Use a generic placeholder since we don't have the filename
+                if len(content) >= MIN_CONTENT_LENGTH:
+                    # Try to extract filename from content if it looks like markdown
+                    # Otherwise use generic placeholder
+                    replacement = _create_shotgun_placeholder("artifact")
+                    logger.debug(
+                        f"Deduplicating .shotgun/ file read ({len(content)} chars)"
+                    )
+
+            # Apply replacement if we have one
+            if replacement:
+                # Create new ToolReturnPart with replaced content
+                new_part = ToolReturnPart(
+                    tool_name=part.tool_name,
+                    tool_call_id=part.tool_call_id,
+                    content=replacement,
+                    timestamp=part.timestamp,
+                )
+                new_parts.append(new_part)
+                message_modified = True
+
+                tokens_saved = _estimate_tokens_saved(original_content, replacement)
+                total_tokens_saved += tokens_saved
+                files_deduplicated += 1
+            else:
+                new_parts.append(part)
+
+        # Replace message with new parts if modified
+        if message_modified:
+            modified_messages[msg_idx] = ModelRequest(parts=new_parts)
+
+    if files_deduplicated > 0:
+        logger.info(
+            f"File content deduplication: {files_deduplicated} files, "
+            f"~{total_tokens_saved:,} tokens saved"
+        )
+
+    return modified_messages, total_tokens_saved