claude-mpm 5.4.48__py3-none-any.whl → 5.6.34__py3-none-any.whl

claude_mpm/services/infrastructure/context_usage_tracker.py

@@ -0,0 +1,291 @@
+"""Context Usage Tracker Service.
+
+WHY: Track cumulative token usage across Claude Code hook invocations to prevent
+context window exhaustion and enable intelligent auto-pause behavior.
+
+DESIGN DECISIONS:
+- File-based persistence (hooks run in separate processes)
+- Atomic file operations using StateStorage
+- Threshold detection at 70% (caution), 85% (warning), 90% (auto_pause), 95% (critical)
+- Session-scoped tracking with reset capability
+- Compatible with 200k context budget for Claude Sonnet 4.5
+
+USAGE:
+    tracker = ContextUsageTracker()
+    state = tracker.update_usage(input_tokens=15000, output_tokens=2000)
+    if tracker.should_auto_pause():
+        # Trigger auto-pause workflow
+        pass
+"""
+
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from claude_mpm.core.logger import get_logger
+from claude_mpm.storage.state_storage import StateStorage
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class ContextUsageState:
+    """State tracking for cumulative context/token usage.
+
+    Attributes:
+        session_id: Unique session identifier
+        cumulative_input_tokens: Total input tokens across all hook invocations
+        cumulative_output_tokens: Total output tokens across all hook invocations
+        cache_creation_tokens: Total tokens spent creating prompt cache
+        cache_read_tokens: Total tokens read from prompt cache
+        percentage_used: Percentage of 200k context budget used (0.0-100.0)
+        threshold_reached: Highest threshold crossed ('caution', 'warning', 'auto_pause', 'critical')
+        auto_pause_active: Whether auto-pause has been triggered
+        last_updated: ISO timestamp of last update
+    """
+
+    session_id: str
+    cumulative_input_tokens: int = 0
+    cumulative_output_tokens: int = 0
+    cache_creation_tokens: int = 0
+    cache_read_tokens: int = 0
+    percentage_used: float = 0.0
+    threshold_reached: Optional[str] = None
+    auto_pause_active: bool = False
+    last_updated: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+
+class ContextUsageTracker:
+    """Track cumulative context/token usage across hook invocations.
+
+    Features:
+    - Cumulative tracking across multiple API calls
+    - File-based persistence for cross-process state sharing
+    - Atomic file operations for concurrent safety
+    - Threshold detection (70%, 85%, 90%, 95%)
+    - Auto-pause triggering at 90%+ usage
+    - Session management with reset capability
+    """
+
+    # Claude Sonnet 4.5 context window
+    CONTEXT_BUDGET = 200000
+
+    # Threshold levels for warnings and auto-pause
+    THRESHOLDS = {
+        "caution": 0.70,  # Yellow warning
+        "warning": 0.85,  # Orange warning
+        "auto_pause": 0.90,  # Trigger auto-pause
+        "critical": 0.95,  # Red critical alert
+    }
+
+    def __init__(self, project_path: Optional[Path] = None):
+        """Initialize context usage tracker.
+
+        Args:
+            project_path: Project root path (default: current directory)
+        """
+        self.project_path = (project_path or Path.cwd()).resolve()
+        self.state_dir = self.project_path / ".claude-mpm" / "state"
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        self.state_file = self.state_dir / "context-usage.json"
+
+        # Use StateStorage for atomic operations
+        self.storage = StateStorage(self.state_dir)
+
+        logger.debug(f"ContextUsageTracker initialized: {self.state_file}")
+
+    def update_usage(
+        self,
+        input_tokens: int,
+        output_tokens: int,
+        cache_creation: int = 0,
+        cache_read: int = 0,
+    ) -> ContextUsageState:
+        """Update cumulative usage from API response.
+
+        Args:
+            input_tokens: Input tokens from this API call
+            output_tokens: Output tokens from this API call
+            cache_creation: Cache creation tokens (optional)
+            cache_read: Cache read tokens (optional)
+
+        Returns:
+            Updated context usage state
+
+        Raises:
+            ValueError: If token counts are negative
+        """
+        if any(
+            t < 0 for t in [input_tokens, output_tokens, cache_creation, cache_read]
+        ):
+            raise ValueError("Token counts cannot be negative")
+
+        # Load current state
+        state = self._load_state()
+
+        # Update cumulative counters
+        state.cumulative_input_tokens += input_tokens
+        state.cumulative_output_tokens += output_tokens
+        state.cache_creation_tokens += cache_creation
+        state.cache_read_tokens += cache_read
+
+        # Calculate total effective tokens (input + output, cache read is "free")
+        total_tokens = state.cumulative_input_tokens + state.cumulative_output_tokens
+        state.percentage_used = (total_tokens / self.CONTEXT_BUDGET) * 100
+
+        # Check thresholds
+        state.threshold_reached = self.check_thresholds(state)
+
+        # Activate auto-pause if threshold reached
+        if state.threshold_reached in {"auto_pause", "critical"}:
+            state.auto_pause_active = True
+
+        # Update timestamp
+        state.last_updated = datetime.now(timezone.utc).isoformat()
+
+        # Persist state atomically
+        self._save_state(state)
+
+        logger.debug(
+            f"Usage updated: {total_tokens}/{self.CONTEXT_BUDGET} tokens "
+            f"({state.percentage_used:.1f}%), threshold: {state.threshold_reached}"
+        )
+
+        return state
+
+    def check_thresholds(
+        self, state: Optional[ContextUsageState] = None
+    ) -> Optional[str]:
+        """Check which threshold (if any) has been exceeded.
+
+        Args:
+            state: Optional state to check (uses current state if None)
+
+        Returns:
+            Highest threshold exceeded ('caution', 'warning', 'auto_pause', 'critical')
+            or None if no thresholds exceeded
+        """
+        if state is None:
+            state = self.get_current_state()
+
+        percentage = state.percentage_used / 100  # Convert to 0.0-1.0
+
+        # Check thresholds in descending order (highest first)
+        for threshold_name in ["critical", "auto_pause", "warning", "caution"]:
+            if percentage >= self.THRESHOLDS[threshold_name]:
+                return threshold_name
+
+        return None
+
+    def should_auto_pause(self) -> bool:
+        """Check if auto-pause should be triggered.
+
+        Returns:
+            True if 90%+ context budget used
+        """
+        state = self.get_current_state()
+        return state.percentage_used >= (self.THRESHOLDS["auto_pause"] * 100)
+
+    def get_current_state(self) -> ContextUsageState:
+        """Get current usage state without modifying.
+
+        Returns:
+            Current context usage state
+        """
+        return self._load_state()
+
+    def reset_session(self, new_session_id: str) -> None:
+        """Reset tracking for a new session.
+
+        Args:
+            new_session_id: New session identifier
+        """
+        state = ContextUsageState(session_id=new_session_id)
+        self._save_state(state)
+        logger.info(f"Context usage reset for new session: {new_session_id}")
+
+    def _load_state(self) -> ContextUsageState:
+        """Load state from persistence file.
+
+        Returns:
+            Loaded state or default state if file doesn't exist/is corrupted
+        """
+        try:
+            if not self.state_file.exists():
+                # Generate initial session ID
+                session_id = (
+                    f"session-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}"
+                )
+                logger.debug("No state file found, creating default state")
+                return ContextUsageState(session_id=session_id)
+
+            # Load JSON state
+            data = self.storage.read_json(self.state_file)
+
+            if data is None:
+                logger.warning("Failed to read state file, using default state")
+                session_id = (
+                    f"session-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}"
+                )
+                return ContextUsageState(session_id=session_id)
+
+            # Reconstruct ContextUsageState from dict
+            return ContextUsageState(**data)
+
+        except Exception as e:
+            logger.error(f"Error loading state, using default: {e}")
+            session_id = (
+                f"session-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}"
+            )
+            return ContextUsageState(session_id=session_id)
+
+    def _save_state(self, state: ContextUsageState) -> None:
+        """Save state to persistence file atomically.
+
+        Args:
+            state: Context usage state to persist
+
+        Raises:
+            RuntimeError: If atomic write fails
+        """
+        try:
+            # Convert dataclass to dict
+            state_dict = asdict(state)
+
+            # Atomic write using StateStorage
+            if not self.storage.write_json(state_dict, self.state_file, atomic=True):
+                raise RuntimeError(f"Failed to write state to {self.state_file}")
+
+            logger.debug(f"State saved: {self.state_file}")
+
+        except Exception as e:
+            logger.error(f"Error saving state: {e}")
+            raise RuntimeError(f"Failed to persist context usage state: {e}") from e
+
+    def get_usage_summary(self) -> dict:
+        """Get human-readable usage summary.
+
+        Returns:
+            Dictionary with usage statistics
+        """
+        state = self.get_current_state()
+        total_tokens = state.cumulative_input_tokens + state.cumulative_output_tokens
+
+        return {
+            "session_id": state.session_id,
+            "total_tokens": total_tokens,
+            "budget": self.CONTEXT_BUDGET,
+            "percentage_used": round(state.percentage_used, 2),
+            "threshold_reached": state.threshold_reached,
+            "auto_pause_active": state.auto_pause_active,
+            "breakdown": {
+                "input_tokens": state.cumulative_input_tokens,
+                "output_tokens": state.cumulative_output_tokens,
+                "cache_creation_tokens": state.cache_creation_tokens,
+                "cache_read_tokens": state.cache_read_tokens,
+            },
+            "last_updated": state.last_updated,
+        }

claude_mpm/services/infrastructure/resume_log_generator.py

@@ -6,7 +6,8 @@ Integrates with session management and response tracking infrastructure.
 Triggers:
 - model_context_window_exceeded (stop_reason)
 - Manual pause command
-- 95% token threshold reached
+- 95% token threshold reached (critical)
+- 90% token threshold reached (auto-pause)
 - Session end with high token usage (>85%)
 
 Design Principles:
@@ -71,6 +72,7 @@ class ResumeLogGenerator:
         thresholds = self.config.get("context_management", {}).get("thresholds", {})
         self.threshold_caution = thresholds.get("caution", 0.70)
         self.threshold_warning = thresholds.get("warning", 0.85)
+        self.threshold_auto_pause = thresholds.get("auto_pause", 0.90)
         self.threshold_critical = thresholds.get("critical", 0.95)
 
         logger.info(
@@ -96,14 +98,14 @@ class ResumeLogGenerator:
         if not self.enabled or not self.auto_generate:
             return manual_trigger  # Only generate on manual trigger if auto is disabled
 
-        # Trigger conditions
+        # Trigger conditions (ordered by severity)
         triggers = [
             stop_reason == "max_tokens",
             stop_reason == "model_context_window_exceeded",
             manual_trigger,
-            token_usage_pct and token_usage_pct >= self.threshold_critical,
-            token_usage_pct
-            and token_usage_pct >= self.threshold_warning,  # Generate at 85% too
+            token_usage_pct and token_usage_pct >= self.threshold_critical,  # 95%
+            token_usage_pct and token_usage_pct >= self.threshold_auto_pause,  # 90%
+            token_usage_pct and token_usage_pct >= self.threshold_warning,  # 85%
         ]
 
         should_gen = any(triggers)
@@ -121,6 +123,22 @@ class ResumeLogGenerator:
 
         return should_gen
 
+    def should_auto_pause(self, token_usage_pct: Optional[float]) -> bool:
+        """Check if auto-pause threshold (90%) has been reached.
+
+        This is a convenience method to check specifically for the 90% threshold
+        which triggers automatic session pausing.
+
+        Args:
+            token_usage_pct: Current token usage percentage (0.0-1.0)
+
+        Returns:
+            True if auto-pause threshold has been reached
+        """
+        if token_usage_pct is None:
+            return False
+        return token_usage_pct >= self.threshold_auto_pause
+
     def generate_from_session_state(
         self,
         session_id: str,
@@ -427,6 +445,7 @@ class ResumeLogGenerator:
                 "thresholds": {
                     "caution": f"{self.threshold_caution:.0%}",
                     "warning": f"{self.threshold_warning:.0%}",
+                    "auto_pause": f"{self.threshold_auto_pause:.0%}",
                     "critical": f"{self.threshold_critical:.0%}",
                 },
             }

claude_mpm/services/monitor/daemon_manager.py

@@ -95,10 +95,10 @@ class DaemonManager:
     def _get_default_log_file(self) -> Path:
         """Get default log file path with port number to support multiple daemons."""
         project_root = Path.cwd()
-        claude_mpm_dir = project_root / ".claude-mpm"
-        claude_mpm_dir.mkdir(exist_ok=True)
+        logs_dir = project_root / ".claude-mpm" / "logs"
+        logs_dir.mkdir(parents=True, exist_ok=True)
         # Include port in filename to support multiple daemon instances
-        return claude_mpm_dir / f"monitor-daemon-{self.port}.log"
+        return logs_dir / f"monitor-daemon-{self.port}.log"
 
     def cleanup_port_conflicts(self, max_retries: int = 3) -> bool:
         """Clean up any processes using the daemon port.
@@ -649,14 +649,24 @@ class DaemonManager:
 
                 # Wait for the subprocess to write its PID file and bind to port
                 # The subprocess will write the PID file after it starts successfully
-                max_wait = 10  # seconds
+                # Allow configuration via environment variable (default 30s to account for agent/skill sync)
+                max_wait = int(os.environ.get("CLAUDE_MPM_MONITOR_TIMEOUT", "30"))
                 start_time = time.time()
                 pid_file_found = False
                 port_bound = False
+                last_progress_log = 0.0
 
                 self.logger.debug(f"Waiting up to {max_wait}s for daemon to start...")
 
                 while time.time() - start_time < max_wait:
+                    # Log progress every 5 seconds to show we're waiting
+                    elapsed = time.time() - start_time
+                    if elapsed - last_progress_log >= 5.0:
+                        self.logger.info(
+                            f"Waiting for monitor daemon... ({int(elapsed)}s elapsed, syncing agents/skills)"
+                        )
+                        last_progress_log = elapsed
+
                     # Check if process is still running
                     returncode = process.poll()
                     if returncode is not None:
@@ -976,6 +986,7 @@ class DaemonManager:
                 os.dup2(null_in.fileno(), sys.stdin.fileno())
 
             # Redirect stdout and stderr to log file
+            # Ensure logs directory exists
             self.log_file.parent.mkdir(parents=True, exist_ok=True)
             with self.log_file.open("a") as log_out:
                 os.dup2(log_out.fileno(), sys.stdout.fileno())

claude_mpm/services/monitor/management/lifecycle.py

@@ -14,7 +14,6 @@ DESIGN DECISIONS:
 """
 
 import json
-import logging.handlers
 import os
 import signal
 import socket
@@ -129,12 +128,16 @@ class DaemonLifecycle:
             # Redirect stdout and stderr
             if self.log_file:
                 # Redirect to log file
+                # Ensure logs directory exists
+                self.log_file.parent.mkdir(parents=True, exist_ok=True)
                 with self.log_file.open("a") as log_out:
                     os.dup2(log_out.fileno(), sys.stdout.fileno())
                     os.dup2(log_out.fileno(), sys.stderr.fileno())
             else:
                 # Default to a daemon log file instead of /dev/null for errors
-                default_log = Path.home() / ".claude-mpm" / "monitor-daemon.log"
+                default_log = (
+                    Path.home() / ".claude-mpm" / "logs" / "monitor-daemon.log"
+                )
                 default_log.parent.mkdir(parents=True, exist_ok=True)
                 with default_log.open("a") as log_out:
                     os.dup2(log_out.fileno(), sys.stdout.fileno())
@@ -476,7 +479,9 @@ class DaemonLifecycle:
         try:
             # If no log file specified, create a default one
             if not self.log_file:
-                default_log = Path.home() / ".claude-mpm" / "monitor-daemon.log"
+                default_log = (
+                    Path.home() / ".claude-mpm" / "logs" / "monitor-daemon.log"
+                )
                 default_log.parent.mkdir(parents=True, exist_ok=True)
                 self.log_file = default_log
 

claude_mpm/services/monitor/server.py

@@ -368,6 +368,83 @@ class UnifiedMonitorServer:
         finally:
             await self._cleanup_async()
 
+    def _categorize_event(self, event_name: str) -> str:
+        """Categorize event by name to determine Socket.IO event type.
+
+        Maps specific event names to their category for frontend filtering.
+
+        Args:
+            event_name: The raw event name (e.g., "subagent_start", "todo_updated")
+
+        Returns:
+            Category name (e.g., "hook_event", "system_event")
+        """
+        # Hook events - agent lifecycle and todo updates
+        if event_name in ("subagent_start", "subagent_stop", "todo_updated"):
+            return "hook_event"
+
+        # Tool events - both hook-style and direct tool events
+        if event_name in (
+            "pre_tool",
+            "post_tool",
+            "tool.start",
+            "tool.end",
+            "tool_use",
+            "tool_result",
+        ):
+            return "tool_event"
+
+        # Session events - session lifecycle
+        if event_name in (
+            "session.started",
+            "session.ended",
+            "session_start",
+            "session_end",
+        ):
+            return "session_event"
+
+        # Response events - API response lifecycle
+        if event_name in (
+            "response.start",
+            "response.end",
+            "response_started",
+            "response_ended",
+        ):
+            return "response_event"
+
+        # Agent events - agent delegation and returns
+        if event_name in (
+            "agent.delegated",
+            "agent.returned",
+            "agent_start",
+            "agent_end",
+        ):
+            return "agent_event"
+
+        # File events - file operations
+        if event_name in (
+            "file.read",
+            "file.write",
+            "file.edit",
+            "file_read",
+            "file_write",
+        ):
+            return "file_event"
+
+        # Claude API events
+        if event_name in ("user_prompt", "assistant_message"):
+            return "claude_event"
+
+        # System events
+        if event_name in ("system_ready", "system_shutdown"):
+            return "system_event"
+
+        # Log uncategorized events for debugging
+        self.logger.debug(f"Uncategorized event: {event_name}")
+
+        # Default to claude_event for unknown events
+        return "claude_event"
+
     def _setup_event_handlers(self):
         """Setup Socket.IO event handlers."""
         try:
@@ -474,7 +551,7 @@ class UnifiedMonitorServer:
                     )
                     if version_file.exists():
                         version = version_file.read_text().strip()
-                except Exception:
+                except Exception:  # nosec B110
                     pass
 
                 return web.json_response(
@@ -499,10 +576,23 @@ class UnifiedMonitorServer:
                     event = data.get("event", "claude_event")
                     event_data = data.get("data", {})
 
-                    # Emit to Socket.IO clients via the appropriate event
+                    # Categorize event and wrap in expected format
+                    event_type = self._categorize_event(event)
+                    wrapped_event = {
+                        "type": event_type,
+                        "subtype": event,
+                        "data": event_data,
+                        "timestamp": event_data.get("timestamp")
+                        or datetime.now(timezone.utc).isoformat() + "Z",
+                        "session_id": event_data.get("session_id"),
+                    }
+
+                    # Emit to Socket.IO clients via the categorized event type
                     if self.sio:
-                        await self.sio.emit(event, event_data)
-                        self.logger.debug(f"HTTP event forwarded to Socket.IO: {event}")
+                        await self.sio.emit(event_type, wrapped_event)
+                        self.logger.debug(
+                            f"HTTP event forwarded to Socket.IO: {event} -> {event_type}"
+                        )
 
                     return web.Response(status=204)  # No content response
 
@@ -859,7 +949,7 @@ class UnifiedMonitorServer:
             # Configuration endpoint for dashboard initialization
             async def config_handler(request):
                 """Return configuration for dashboard initialization."""
-                import subprocess
+                import subprocess  # nosec B404
 
                 config = {
                     "workingDirectory": Path.cwd(),
@@ -870,7 +960,7 @@ class UnifiedMonitorServer:
 
                 # Try to get current git branch
                 try:
-                    result = subprocess.run(
+                    result = subprocess.run(  # nosec B603 B607
                         ["git", "branch", "--show-current"],
                         capture_output=True,
                         text=True,
@@ -880,7 +970,7 @@ class UnifiedMonitorServer:
                     )
                     if result.returncode == 0 and result.stdout.strip():
                         config["gitBranch"] = result.stdout.strip()
-                except Exception:
+                except Exception:  # nosec B110
                     pass  # Keep default "Unknown" value
 
                 return web.json_response(config)
@@ -910,7 +1000,7 @@ class UnifiedMonitorServer:
             # Git history handler
             async def git_history_handler(request: web.Request) -> web.Response:
                 """Get git history for a file."""
-                import subprocess
+                import subprocess  # nosec B404
 
                 try:
                     data = await request.json()
@@ -939,7 +1029,7 @@ class UnifiedMonitorServer:
                         )
 
                     # Get git log for file
-                    result = subprocess.run(
+                    result = subprocess.run(  # nosec B603 B607
                         [
                             "git",
                             "log",
@@ -978,7 +1068,7 @@ class UnifiedMonitorServer:
             # Git diff handler
             async def git_diff_handler(request: web.Request) -> web.Response:
                 """Get git diff for a file with optional commit selection."""
-                import subprocess
+                import subprocess  # nosec B404
 
                 try:
                     file_path = request.query.get("path", "")
@@ -1010,7 +1100,7 @@ class UnifiedMonitorServer:
                         )
 
                     # Find git repository root
-                    git_root_result = subprocess.run(
+                    git_root_result = subprocess.run(  # nosec B603 B607
                         ["git", "rev-parse", "--show-toplevel"],
                         check=False,
                         capture_output=True,
@@ -1034,7 +1124,7 @@ class UnifiedMonitorServer:
                     git_root = Path(git_root_result.stdout.strip())
 
                     # Check if file is tracked by git
-                    ls_files_result = subprocess.run(
+                    ls_files_result = subprocess.run(  # nosec B603 B607
                         ["git", "ls-files", "--error-unmatch", str(path)],
                         check=False,
                         capture_output=True,
@@ -1056,7 +1146,7 @@ class UnifiedMonitorServer:
                         )
 
                     # Get commit history for this file (last 5 commits)
-                    history_result = subprocess.run(
+                    history_result = subprocess.run(  # nosec B603 B607
                         [
                             "git",
                             "log",
@@ -1087,7 +1177,7 @@ class UnifiedMonitorServer:
                                     )
 
                     # Check for uncommitted changes
-                    uncommitted_result = subprocess.run(
+                    uncommitted_result = subprocess.run(  # nosec B603 B607
                         ["git", "diff", "HEAD", str(path)],
                         check=False,
                         capture_output=True,
@@ -1100,7 +1190,7 @@ class UnifiedMonitorServer:
                     # Get diff based on commit parameter
                     if commit_hash:
                         # Get diff for specific commit
-                        result = subprocess.run(
+                        result = subprocess.run(  # nosec B603 B607
                             ["git", "show", commit_hash, "--", str(path)],
                             check=False,
                             capture_output=True,
@@ -1469,7 +1559,7 @@ class UnifiedMonitorServer:
                 gather = asyncio.gather(*tasks_to_cancel, return_exceptions=True)
                 try:
                     loop.run_until_complete(gather)
-                except Exception:
+                except Exception:  # nosec B110
                     # Some tasks might fail to cancel, that's ok
                     pass