gobby 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

gobby/conductor/loop.py

@@ -0,0 +1,164 @@
+"""Conductor loop for orchestrating monitors and agents.
+
+The main daemon loop that:
+- Runs TaskMonitor and AgentWatcher periodically
+- Dispatches alerts based on monitor results
+- Checks budget before running
+- Optionally spawns agents in autonomous mode
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+    from gobby.conductor.alerts import AlertDispatcher
+    from gobby.conductor.monitors.agents import AgentWatcher
+    from gobby.conductor.monitors.tasks import TaskMonitor
+
+logger = logging.getLogger(__name__)
+
+
+class BudgetChecker(Protocol):
+    """Protocol for budget checking."""
+
+    def is_budget_exceeded(self) -> bool:
+        """Check if budget is exceeded."""
+        ...
+
+
+class AgentSpawner(Protocol):
+    """Protocol for agent spawning."""
+
+    def spawn(self, task_id: str) -> dict[str, Any]:
+        """Spawn an agent for a task."""
+        ...
+
+
+@dataclass
+class ConductorLoop:
+    """Main conductor loop that orchestrates monitors and agents.
+
+    Runs periodic checks:
+    - TaskMonitor: Detects stale tasks and blocked chains
+    - AgentWatcher: Detects stuck agents
+    - AlertDispatcher: Sends alerts for issues
+
+    Supports optional autonomous mode for auto-spawning agents.
+    """
+
+    task_monitor: TaskMonitor
+    """Monitor for task health."""
+
+    agent_watcher: AgentWatcher
+    """Watcher for agent health."""
+
+    alert_dispatcher: AlertDispatcher
+    """Dispatcher for alerts."""
+
+    budget_checker: BudgetChecker | None = None
+    """Optional budget checker for throttling."""
+
+    agent_spawner: AgentSpawner | None = None
+    """Optional agent spawner for autonomous mode."""
+
+    autonomous_mode: bool = False
+    """Whether to auto-spawn agents for ready tasks."""
+
+    _logger: logging.Logger = field(default_factory=lambda: logging.getLogger(__name__))
+    """Logger instance."""
+
+    def tick(self) -> dict[str, Any]:
+        """
+        Run one iteration of the conductor loop.
+
+        This method:
+        1. Checks budget (if budget_checker configured)
+        2. Runs TaskMonitor.check()
+        3. Runs AgentWatcher.check()
+        4. Dispatches alerts for any issues found
+        5. Optionally spawns agents in autonomous mode
+
+        Returns:
+            Dict with results from all checks and alerts
+        """
+        now = datetime.now(UTC)
+
+        # Check budget first
+        if self.budget_checker is not None:
+            if self.budget_checker.is_budget_exceeded():
+                self._logger.warning("Budget exceeded, throttling conductor")
+                return {
+                    "success": True,
+                    "throttled": True,
+                    "reason": "budget_exceeded",
+                    "checked_at": now.isoformat(),
+                }
+
+        # Run monitors
+        task_result = self.task_monitor.check()
+        agent_result = self.agent_watcher.check()
+
+        # Track alerts dispatched
+        alerts_dispatched = []
+
+        # Alert for stale tasks
+        stale_count = task_result["summary"]["stale_count"]
+        if stale_count > 0:
+            alert_result = self.alert_dispatcher.dispatch(
+                priority="urgent",
+                message=f"{stale_count} stale task(s) detected",
+                context={"stale_tasks": task_result["stale_tasks"]},
+                source="TaskMonitor",
+            )
+            alerts_dispatched.append(alert_result)
+
+        # Alert for blocked chains
+        blocked_count = task_result["summary"]["blocked_count"]
+        if blocked_count > 0:
+            alert_result = self.alert_dispatcher.dispatch(
+                priority="info",
+                message=f"{blocked_count} blocked task chain(s) detected",
+                context={"blocked_chains": task_result["blocked_chains"]},
+                source="TaskMonitor",
+            )
+            alerts_dispatched.append(alert_result)
+
+        # Alert for stuck agents
+        stuck_count = agent_result["summary"]["stuck_count"]
+        if stuck_count > 0:
+            alert_result = self.alert_dispatcher.dispatch(
+                priority="urgent",
+                message=f"{stuck_count} stuck agent(s) detected",
+                context={"stuck_agents": agent_result["stuck_agents"]},
+                source="AgentWatcher",
+            )
+            alerts_dispatched.append(alert_result)
+
+        # Build result
+        result: dict[str, Any] = {
+            "success": True,
+            "task_monitor_result": task_result,
+            "agent_watcher_result": agent_result,
+            "alerts_dispatched": len(alerts_dispatched),
+            "checked_at": now.isoformat(),
+        }
+
+        # Handle autonomous mode
+        if self.autonomous_mode:
+            result["autonomous_mode"] = True
+            if self.agent_spawner is not None:
+                result["spawner_available"] = True
+                # TODO: implement auto-spawn - see issue tracker for orchestration epic
+                self._logger.warning(
+                    "Autonomous mode enabled but auto-spawning not yet implemented. "
+                    f"Spawner available: {self.agent_spawner is not None}"
+                )
+            else:
+                result["spawner_available"] = False
+                self._logger.warning("Autonomous mode enabled but no spawner configured")
+
+        return result

gobby/conductor/monitors/__init__.py

@@ -0,0 +1,11 @@
+"""Conductor monitors for task and system health.
+
+Monitors detect issues that need attention:
+- TaskMonitor: Stale tasks, blocked chains
+- AgentWatcher: Stuck agents
+"""
+
+from gobby.conductor.monitors.agents import AgentWatcher
+from gobby.conductor.monitors.tasks import TaskMonitor
+
+__all__ = ["AgentWatcher", "TaskMonitor"]

gobby/conductor/monitors/agents.py

@@ -0,0 +1,116 @@
+"""Agent watcher for detecting stuck agents.
+
+Provides monitoring for:
+- Stuck agents: Running longer than threshold without progress
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from gobby.agents.registry import RunningAgentRegistry
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class AgentWatcher:
+    """Watcher for agent health and status.
+
+    Detects:
+    - Stuck agents: Running longer than threshold
+    """
+
+    agent_registry: RunningAgentRegistry
+
+    def check(
+        self,
+        stuck_threshold_minutes: int = 15,
+        mode: str | None = None,
+    ) -> dict[str, Any]:
+        """
+        Check for agent health issues.
+
+        Args:
+            stuck_threshold_minutes: Minutes before an agent is considered stuck
+            mode: Optional filter by agent mode (terminal, headless, etc.)
+
+        Returns:
+            Dict with stuck_agents and summary
+        """
+        stuck_agents = self._find_stuck_agents(
+            threshold_minutes=stuck_threshold_minutes,
+            mode=mode,
+        )
+
+        # Get all running agents for total count
+        all_agents = self.agent_registry.list_all()
+        if mode:
+            all_agents = [a for a in all_agents if a.mode == mode]
+
+        return {
+            "stuck_agents": stuck_agents,
+            "summary": {
+                "stuck_count": len(stuck_agents),
+                "total_running": len(all_agents),
+                "checked_at": datetime.now(UTC).isoformat(),
+            },
+        }
+
+    def _find_stuck_agents(
+        self,
+        threshold_minutes: int = 15,
+        mode: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Find agents that have been running longer than threshold.
+
+        Args:
+            threshold_minutes: Minutes before agent is considered stuck
+            mode: Optional filter by agent mode
+
+        Returns:
+            List of stuck agent info dicts
+        """
+        all_agents = self.agent_registry.list_all()
+
+        # Apply mode filter if specified
+        if mode:
+            all_agents = [a for a in all_agents if a.mode == mode]
+
+        threshold = datetime.now(UTC) - timedelta(minutes=threshold_minutes)
+        stuck_agents = []
+
+        for agent in all_agents:
+            # Check if agent has been running longer than threshold
+            started_at = agent.started_at
+
+            # Timezone policy: All timestamps are expected to be in UTC.
+            # If started_at is naive (no tzinfo), log a warning and skip this agent
+            # rather than assume UTC, as the source may be using local time.
+            if started_at.tzinfo is None:
+                logger.warning(
+                    f"Agent {agent.run_id} has naive started_at timestamp "
+                    f"({started_at}); skipping stuck detection. "
+                    "Ensure the agent registry stores UTC timestamps."
+                )
+                continue
+
+            if started_at < threshold:
+                minutes_running = (datetime.now(UTC) - started_at).total_seconds() / 60
+                stuck_agents.append(
+                    {
+                        "run_id": agent.run_id,
+                        "session_id": agent.session_id,
+                        "mode": agent.mode,
+                        "started_at": started_at.isoformat(),
+                        "minutes_running": round(minutes_running, 1),
+                        "provider": getattr(agent, "provider", "unknown"),
+                    }
+                )
+
+        return stuck_agents

gobby/conductor/monitors/tasks.py

@@ -0,0 +1,155 @@
+"""Task monitor for detecting stale and blocked tasks.
+
+Provides monitoring for:
+- Stale tasks: Tasks in_progress longer than a threshold
+- Blocked chains: Tasks blocked by open dependencies
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from gobby.storage.tasks import LocalTaskManager
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TaskMonitor:
+    """Monitor for task health and status.
+
+    Detects:
+    - Stale tasks: in_progress longer than threshold
+    - Blocked chains: tasks waiting on open dependencies
+    """
+
+    task_manager: LocalTaskManager
+
+    def check(
+        self,
+        project_id: str | None = None,
+        stale_threshold_hours: int = 24,
+    ) -> dict[str, Any]:
+        """
+        Check for task health issues.
+
+        Args:
+            project_id: Optional project filter
+            stale_threshold_hours: Hours before an in_progress task is considered stale
+
+        Returns:
+            Dict with stale_tasks, blocked_chains, and summary
+        """
+        stale_tasks = self._find_stale_tasks(
+            project_id=project_id,
+            threshold_hours=stale_threshold_hours,
+        )
+        blocked_chains = self._find_blocked_chains(project_id=project_id)
+
+        return {
+            "stale_tasks": stale_tasks,
+            "blocked_chains": blocked_chains,
+            "summary": {
+                "stale_count": len(stale_tasks),
+                "blocked_count": len(blocked_chains),
+                "checked_at": datetime.now(UTC).isoformat(),
+            },
+        }
+
+    def _find_stale_tasks(
+        self,
+        project_id: str | None = None,
+        threshold_hours: int = 24,
+    ) -> list[dict[str, Any]]:
+        """
+        Find tasks that have been in_progress longer than threshold.
+
+        Args:
+            project_id: Optional project filter
+            threshold_hours: Hours before task is considered stale
+
+        Returns:
+            List of stale task info dicts
+        """
+        # Get all in_progress tasks
+        in_progress_tasks = self.task_manager.list_tasks(
+            project_id=project_id,
+            status="in_progress",
+            limit=1000,
+        )
+
+        threshold = datetime.now(UTC) - timedelta(hours=threshold_hours)
+        stale_tasks = []
+
+        for task in in_progress_tasks:
+            # Parse updated_at timestamp
+            try:
+                if task.updated_at:
+                    # Handle both string and datetime types
+                    if isinstance(task.updated_at, str):
+                        # Parse ISO format, handle both Z and +00:00 formats
+                        updated_str = task.updated_at.replace("Z", "+00:00")
+                        updated_at = datetime.fromisoformat(updated_str)
+                    else:
+                        updated_at = task.updated_at
+
+                    # Timezone policy: All timestamps are expected to be stored in UTC.
+                    # If updated_at is naive (no tzinfo), log a warning and skip
+                    # rather than assuming UTC which could cause incorrect staleness detection.
+                    if updated_at.tzinfo is None:
+                        logger.warning(
+                            f"Task {task.id} has naive updated_at timestamp "
+                            f"({updated_at}); skipping staleness check. "
+                            "Ensure the task storage stores UTC timestamps."
+                        )
+                        continue
+
+                    if updated_at < threshold:
+                        hours_stale = (datetime.now(UTC) - updated_at).total_seconds() / 3600
+                        stale_tasks.append(
+                            {
+                                "task_id": task.id,
+                                "title": task.title,
+                                "updated_at": task.updated_at,
+                                "hours_stale": round(hours_stale, 1),
+                            }
+                        )
+            except (ValueError, TypeError) as e:
+                logger.warning(f"Could not parse updated_at for task {task.id}: {e}")
+                continue
+
+        return stale_tasks
+
+    def _find_blocked_chains(
+        self,
+        project_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Find blocked task chains.
+
+        Args:
+            project_id: Optional project filter
+
+        Returns:
+            List of blocked chain info dicts
+        """
+        blocked_tasks = self.task_manager.list_blocked_tasks(
+            project_id=project_id,
+            limit=1000,
+        )
+
+        blocked_chains = []
+        for task in blocked_tasks:
+            blocked_chains.append(
+                {
+                    "task_id": task.id,
+                    "title": task.title,
+                    "status": task.status,
+                }
+            )
+
+        return blocked_chains

gobby/conductor/pricing.py

@@ -0,0 +1,234 @@
+"""Token tracking and cost calculation using LiteLLM pricing utilities.
+
+LiteLLM maintains model_prices_and_context_window.json with current pricing
+for 100+ models, so we don't need to maintain our own pricing data.
+
+See: https://docs.litellm.ai/docs/completion/token_usage
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+try:
+    import litellm
+except ImportError:
+    litellm = None  # type: ignore[assignment]
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TokenTracker:
+    """Track token usage and calculate costs using LiteLLM pricing utilities.
+
+    LiteLLM automatically maintains pricing data for 100+ models including:
+    - Claude models (Anthropic)
+    - GPT models (OpenAI)
+    - Gemini models (Google)
+    - And many more
+
+    Example:
+        tracker = TokenTracker()
+        cost = tracker.calculate_cost("claude-3-5-sonnet", 1000, 500)
+        tracker.track_usage("claude-3-5-sonnet", 1000, 500)
+        print(tracker.get_summary())
+    """
+
+    # Accumulated token counts
+    total_input_tokens: int = 0
+    total_output_tokens: int = 0
+    total_cache_read_tokens: int = 0
+    total_cache_write_tokens: int = 0
+    total_cost: float = 0.0
+
+    # Track usage by model
+    usage_by_model: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+    def calculate_cost(
+        self,
+        model: str,
+        input_tokens: int,
+        output_tokens: int,
+        cache_read_tokens: int = 0,
+        cache_write_tokens: int = 0,
+    ) -> float:
+        """Calculate cost for given token usage using LiteLLM pricing.
+
+        Uses litellm.cost_per_token() to get per-token pricing for the model,
+        then calculates total cost.
+
+        Args:
+            model: Model name (e.g., "claude-3-5-sonnet-20241022", "gpt-4o")
+            input_tokens: Number of input (prompt) tokens
+            output_tokens: Number of output (completion) tokens
+            cache_read_tokens: Number of cache read tokens (if model supports)
+            cache_write_tokens: Number of cache write tokens (if model supports)
+
+        Returns:
+            Total cost in USD, or 0.0 if pricing is unavailable
+        """
+        if input_tokens == 0 and output_tokens == 0:
+            return 0.0
+
+        if litellm is None:
+            logger.debug("litellm not available, cannot calculate cost")
+            return 0.0
+
+        try:
+            # Get cost per token for this model
+            prompt_cost, completion_cost = litellm.cost_per_token(
+                model=model,
+                prompt_tokens=input_tokens,
+                completion_tokens=output_tokens,
+            )
+
+            total = prompt_cost + completion_cost
+
+            # Handle cache tokens if provided using LiteLLM's native cache pricing
+            if cache_read_tokens > 0 or cache_write_tokens > 0:
+                try:
+                    # Get model cost info from LiteLLM
+                    model_info = litellm.get_model_info(model=model)
+
+                    # Check for cache-specific pricing in model info
+                    cache_read_cost_per_token = model_info.get("cache_read_input_token_cost")
+                    cache_creation_cost_per_token = model_info.get(
+                        "cache_creation_input_token_cost"
+                    )
+
+                    if cache_read_tokens > 0:
+                        if cache_read_cost_per_token is not None:
+                            # Use native cache read pricing
+                            total += cache_read_tokens * cache_read_cost_per_token
+                        else:
+                            # Fallback: cache reads are typically 10% of input cost
+                            input_cost_per_token = model_info.get("input_cost_per_token", 0)
+                            total += cache_read_tokens * input_cost_per_token * 0.1
+
+                    if cache_write_tokens > 0:
+                        if cache_creation_cost_per_token is not None:
+                            # Use native cache creation pricing
+                            total += cache_write_tokens * cache_creation_cost_per_token
+                        else:
+                            # Fallback: cache writes are typically 1.25x input cost
+                            input_cost_per_token = model_info.get("input_cost_per_token", 0)
+                            total += cache_write_tokens * input_cost_per_token * 1.25
+
+                    # Note: For Anthropic models with prompt caching, LiteLLM may
+                    # already account for cache tokens in cost_per_token. Check if
+                    # the response usage includes cached_tokens to avoid double-counting.
+
+                except Exception:  # nosec B110 - best effort cache pricing, failure is non-critical
+                    # If cache pricing lookup fails, skip cache cost calculation
+                    pass
+
+            return total
+
+        except Exception as e:
+            # Model not found in LiteLLM pricing data
+            logger.debug(f"Could not calculate cost for model {model}: {e}")
+            return 0.0
+
+    def calculate_cost_from_response(self, response: Any) -> float:
+        """Calculate cost directly from a LiteLLM response object.
+
+        Uses litellm.completion_cost() which extracts usage info from the response
+        and calculates the total cost.
+
+        Args:
+            response: LiteLLM response object from acompletion/completion
+
+        Returns:
+            Total cost in USD, or 0.0 if calculation fails
+        """
+        if litellm is None:
+            logger.debug("litellm not available, cannot calculate cost from response")
+            return 0.0
+
+        try:
+            return litellm.completion_cost(response)
+        except Exception as e:
+            logger.debug(f"Could not calculate cost from response: {e}")
+            return 0.0
+
+    def track_usage(
+        self,
+        model: str,
+        input_tokens: int,
+        output_tokens: int,
+        cache_read_tokens: int = 0,
+        cache_write_tokens: int = 0,
+    ) -> float:
+        """Track token usage and accumulate costs.
+
+        Args:
+            model: Model name
+            input_tokens: Number of input tokens
+            output_tokens: Number of output tokens
+            cache_read_tokens: Number of cache read tokens
+            cache_write_tokens: Number of cache write tokens
+
+        Returns:
+            Cost for this usage
+        """
+        cost = self.calculate_cost(
+            model=model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            cache_read_tokens=cache_read_tokens,
+            cache_write_tokens=cache_write_tokens,
+        )
+
+        # Accumulate totals
+        self.total_input_tokens += input_tokens
+        self.total_output_tokens += output_tokens
+        self.total_cache_read_tokens += cache_read_tokens
+        self.total_cache_write_tokens += cache_write_tokens
+        self.total_cost += cost
+
+        # Track by model
+        if model not in self.usage_by_model:
+            self.usage_by_model[model] = {
+                "input_tokens": 0,
+                "output_tokens": 0,
+                "cache_read_tokens": 0,
+                "cache_write_tokens": 0,
+                "cost": 0.0,
+                "calls": 0,
+            }
+
+        self.usage_by_model[model]["input_tokens"] += input_tokens
+        self.usage_by_model[model]["output_tokens"] += output_tokens
+        self.usage_by_model[model]["cache_read_tokens"] += cache_read_tokens
+        self.usage_by_model[model]["cache_write_tokens"] += cache_write_tokens
+        self.usage_by_model[model]["cost"] += cost
+        self.usage_by_model[model]["calls"] += 1
+
+        return cost
+
+    def reset(self) -> None:
+        """Reset all tracked usage."""
+        self.total_input_tokens = 0
+        self.total_output_tokens = 0
+        self.total_cache_read_tokens = 0
+        self.total_cache_write_tokens = 0
+        self.total_cost = 0.0
+        self.usage_by_model.clear()
+
+    def get_summary(self) -> dict[str, Any]:
+        """Get a summary of all tracked usage.
+
+        Returns:
+            Dict with total tokens, cost, and per-model breakdown
+        """
+        return {
+            "total_input_tokens": self.total_input_tokens,
+            "total_output_tokens": self.total_output_tokens,
+            "total_cache_read_tokens": self.total_cache_read_tokens,
+            "total_cache_write_tokens": self.total_cache_write_tokens,
+            "total_cost": self.total_cost,
+            "usage_by_model": dict(self.usage_by_model),
+        }