monoco-toolkit 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

monoco/daemon/mailroom_service.py

@@ -0,0 +1,196 @@
+"""
+Mailroom Service for Monoco Daemon.
+
+Manages automated document ingestion with concurrent processing,
+environment discovery, and artifact registration.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+from monoco.core.ingestion.discovery import EnvironmentDiscovery
+from monoco.core.ingestion.worker import ConversionWorker
+from monoco.core.ingestion.watcher import DropzoneWatcher, IngestionEvent
+from monoco.core.artifacts.manager import ArtifactManager
+
+logger = logging.getLogger(__name__)
+
+
+class MailroomService:
+    """
+    Service for automated document ingestion in Monoco Daemon.
+    
+    Features:
+    - Environment discovery for conversion tools
+    - Dropzone monitoring for new files
+    - Concurrent conversion processing
+    - Artifact registration
+    - SSE event broadcasting
+    """
+
+    def __init__(
+        self,
+        workspace_root: Path,
+        broadcaster: Optional[Any] = None,
+        dropzone_path: Optional[Path] = None,
+        max_concurrent: int = 4,
+    ):
+        """
+        Initialize the Mailroom service.
+        
+        Args:
+            workspace_root: Root directory of the workspace
+            broadcaster: SSE broadcaster for events
+            dropzone_path: Path to dropzone directory (default: workspace/.monoco/dropzone)
+            max_concurrent: Maximum concurrent conversion tasks
+        """
+        self.workspace_root = Path(workspace_root)
+        self.broadcaster = broadcaster
+        
+        # Default dropzone location
+        self.dropzone_path = dropzone_path or (self.workspace_root / ".monoco" / "dropzone")
+        
+        # Initialize components
+        self.discovery = EnvironmentDiscovery()
+        self.conversion_worker = ConversionWorker(
+            discovery=self.discovery,
+            max_concurrent=max_concurrent,
+        )
+        
+        # Artifact manager (lazy init)
+        self._artifact_manager: Optional[ArtifactManager] = None
+        
+        # Watcher (lazy init)
+        self._watcher: Optional[DropzoneWatcher] = None
+        
+        # State
+        self._running = False
+        self._stats: Dict[str, Any] = {
+            "files_detected": 0,
+            "conversions_success": 0,
+            "conversions_failed": 0,
+            "artifacts_registered": 0,
+        }
+
+    @property
+    def artifact_manager(self) -> ArtifactManager:
+        """Get or create the artifact manager."""
+        if self._artifact_manager is None:
+            self._artifact_manager = ArtifactManager(self.workspace_root)
+        return self._artifact_manager
+
+    async def start(self) -> None:
+        """Start the Mailroom service."""
+        if self._running:
+            return
+
+        logger.info("Starting Mailroom service...")
+
+        # Perform environment discovery
+        tools = self.discovery.discover()
+        total_tools = sum(len(t) for t in tools.values())
+        logger.info(f"Discovered {total_tools} conversion tools")
+
+        # Log discovered capabilities
+        capabilities = self.discovery.get_capabilities_summary()
+        logger.info(f"Capabilities: {capabilities}")
+
+        # Initialize and start dropzone watcher
+        self._watcher = DropzoneWatcher(
+            dropzone_path=self.dropzone_path,
+            artifact_manager=self.artifact_manager,
+            conversion_worker=self.conversion_worker,
+            process_existing=False,  # Don't process existing files on startup
+        )
+        
+        # Set up event callback
+        self._watcher.set_event_callback(self._on_ingestion_event)
+        
+        # Start watching (this is synchronous, runs in background thread)
+        self._watcher.start()
+        
+        self._running = True
+        logger.info(f"Mailroom service started. Dropzone: {self.dropzone_path}")
+
+    async def stop(self) -> None:
+        """Stop the Mailroom service."""
+        if not self._running:
+            return
+
+        logger.info("Stopping Mailroom service...")
+
+        if self._watcher:
+            self._watcher.stop()
+            self._watcher = None
+
+        self._running = False
+        logger.info("Mailroom service stopped")
+
+    def _on_ingestion_event(self, event: IngestionEvent) -> None:
+        """Handle ingestion events from the watcher."""
+        # Update stats
+        if event.event_type.value == "file_detected":
+            self._stats["files_detected"] += 1
+        elif event.event_type.value == "conversion_completed":
+            self._stats["conversions_success"] += 1
+        elif event.event_type.value == "conversion_failed":
+            self._stats["conversions_failed"] += 1
+        elif event.event_type.value == "artifact_registered":
+            self._stats["artifacts_registered"] += 1
+
+        # Broadcast via SSE if broadcaster available
+        if self.broadcaster:
+            asyncio.create_task(self._broadcast_event(event))
+
+    async def _broadcast_event(self, event: IngestionEvent) -> None:
+        """Broadcast ingestion event to SSE clients."""
+        try:
+            payload = {
+                "type": event.event_type.value,
+                "file_path": str(event.file_path),
+                "task_id": event.task_id,
+                "artifact_id": event.artifact_id,
+                "error_message": event.error_message,
+                "metadata": event.metadata,
+                "timestamp": event.timestamp.isoformat(),
+            }
+            await self.broadcaster.broadcast("MAILROOM_EVENT", payload)
+        except Exception as e:
+            logger.error(f"Failed to broadcast mailroom event: {e}")
+
+    def get_status(self) -> Dict[str, Any]:
+        """Get current service status and statistics."""
+        capabilities = self.discovery.get_capabilities_summary()
+        
+        return {
+            "running": self._running,
+            "dropzone_path": str(self.dropzone_path),
+            "capabilities": capabilities,
+            "supported_extensions": self.conversion_worker.get_supported_extensions(),
+            "stats": self._stats.copy(),
+            "tools": [
+                {
+                    "name": tool.name,
+                    "type": tool.tool_type.value,
+                    "version": tool.version,
+                    "capabilities": [c.value for c in tool.capabilities],
+                }
+                for tool in self.discovery.get_all_tools()
+            ],
+        }
+
+    def is_running(self) -> bool:
+        """Check if the service is running."""
+        return self._running
+
+    def get_discovery(self) -> EnvironmentDiscovery:
+        """Get the environment discovery instance."""
+        return self.discovery
+
+    def get_worker(self) -> ConversionWorker:
+        """Get the conversion worker instance."""
+        return self.conversion_worker

monoco/daemon/models.py

@@ -18,6 +18,7 @@ class CreateIssueRequest(BaseModel):
     related: List[str] = []
     subdir: Optional[str] = None
     project_id: Optional[str] = None  # Added for multi-project support
+    from_memos: List[str] = []  # Memo IDs to link to this issue
 
 
 class UpdateIssueRequest(BaseModel):

monoco/daemon/scheduler.py

@@ -0,0 +1,236 @@
+import asyncio
+import logging
+import os
+from typing import Dict, Optional, List, Any, Tuple
+from pathlib import Path
+
+from monoco.daemon.services import ProjectManager, SemaphoreManager
+from monoco.daemon.triggers import MemoAccumulationPolicy, HandoverPolicy
+from monoco.features.agent.manager import SessionManager
+from monoco.features.agent.models import RoleTemplate
+from monoco.features.agent.session import RuntimeSession
+from monoco.features.agent.apoptosis import ApoptosisManager
+from monoco.features.issue.core import list_issues
+from monoco.core.config import get_config
+
+logger = logging.getLogger("monoco.daemon.scheduler")
+
+class SchedulerService:
+    def __init__(self, project_manager: ProjectManager):
+        self.project_manager = project_manager
+        self.session_managers: Dict[str, SessionManager] = {} 
+        self._monitoring_task: Optional[asyncio.Task] = None
+        self.apoptosis_managers: Dict[str, ApoptosisManager] = {}
+        
+        # Initialize SemaphoreManager with config
+        config = self._load_concurrency_config()
+        self.semaphore_manager = SemaphoreManager(config)
+
+    def _load_concurrency_config(self) -> Optional[Any]:
+        """Load concurrency configuration from config files and env vars."""
+        try:
+            settings = get_config()
+            concurrency_config = settings.agent.concurrency
+            
+            # Check for environment variable override
+            env_max_agents = os.environ.get("MONOCO_MAX_AGENTS")
+            if env_max_agents:
+                try:
+                    concurrency_config.global_max = int(env_max_agents)
+                    logger.info(f"Overriding global_max from environment: {env_max_agents}")
+                except ValueError:
+                    logger.warning(f"Invalid MONOCO_MAX_AGENTS value: {env_max_agents}")
+            
+            return concurrency_config
+        except Exception as e:
+            logger.warning(f"Failed to load concurrency config: {e}. Using defaults.")
+            return None
+
+    def get_managers(self, project_path: Path) -> Tuple[SessionManager, ApoptosisManager]:
+        key = str(project_path)
+        if key not in self.session_managers:
+            sm = SessionManager(project_root=project_path)
+            self.session_managers[key] = sm
+            self.apoptosis_managers[key] = ApoptosisManager(sm)
+        return self.session_managers[key], self.apoptosis_managers[key]
+
+    async def start(self):
+        logger.info("Starting Scheduler Service...")
+        self._monitoring_task = asyncio.create_task(self.monitor_loop())
+
+    def stop(self):
+        logger.info("Stopping Scheduler Service...")
+        if self._monitoring_task:
+            self._monitoring_task.cancel()
+        
+        # Terminate all sessions
+        for sm in self.session_managers.values():
+            filtered_sessions = sm.list_sessions() 
+            for session in filtered_sessions:
+                session.terminate()
+
+    async def monitor_loop(self):
+        try:
+            while True:
+                await self.tick()
+                await asyncio.sleep(5)
+        except asyncio.CancelledError:
+            pass
+        except Exception as e:
+            logger.error(f"Scheduler loop crashed: {e}", exc_info=True)
+
+    async def tick(self):
+        # We iterate over keys to avoid modification during iteration issues if new projects added
+        projects = list(self.project_manager.projects.values())
+        for project_ctx in projects:
+            await self.process_project(project_ctx)
+
+    async def process_project(self, project_context):
+        sm, am = self.get_managers(project_context.path)
+        
+        # 1. Trigger Check: Architect
+        await self.check_inbox_trigger(sm, project_context)
+
+        # 1.5 Handover Trigger: Architect -> Engineer
+        await self.check_handover_trigger(sm, project_context)
+
+        # 2. Monitor Active Sessions (Supervisor)
+        active_sessions = sm.list_sessions()
+        for session in active_sessions:
+            if session.model.status in ["running", "pending"]:
+                status = session.refresh_status() # Updates model.status
+                
+                # Check for timeout/failure
+                if status == "timeout" or status == "failed":
+                     if session.model.status != "crashed": 
+                         logger.warning(f"Session {session.model.id} led to {status}. Triggering Autopsy.")
+                         # Record failure for cooldown
+                         self.semaphore_manager.record_failure(
+                             issue_id=session.model.issue_id,
+                             session_id=session.model.id
+                         )
+                         am.trigger_apoptosis(session.model.id, failure_reason=f"Session status became {status}")
+                else:
+                    # Track active session in semaphore manager
+                    self.semaphore_manager.acquire(session.model.id, session.model.role_name)
+                
+                # Daemon Logic for Chained Execution
+                if status == "completed":
+                    # Clear failure record on success
+                    self.semaphore_manager.clear_failure(session.model.issue_id)
+                    self.handle_completion(session, sm)
+
+    async def check_inbox_trigger(self, sm: SessionManager, project_context):
+        # Checking existing Architect sessions
+        existing_architects = [s for s in sm.list_sessions() if s.model.role_name == "Architect" and s.model.status == "running"]
+        
+        if not existing_architects:
+            # Check semaphore before spawning
+            if not self.semaphore_manager.can_acquire("Architect"):
+                logger.warning("Cannot spawn Architect: concurrency limit reached")
+                return
+                
+            trigger_policy = MemoAccumulationPolicy(count_threshold=5)
+            if trigger_policy.evaluate({"issues_root": project_context.issues_root}):
+                logger.info(f"Triggering Architect for project {project_context.id}")
+                self.spawn_architect(sm, project_context)
+
+    async def check_handover_trigger(self, sm: SessionManager, project_context):
+        # Scan for OPEN + DOING issues with no active worker
+        try:
+            all_issues = list_issues(project_context.issues_root)
+            handover_policy = HandoverPolicy(target_status="open", target_stage="doing")
+            
+            for issue in all_issues:
+                if handover_policy.evaluate({"issue": issue}):
+                    # Check if session exists
+                    active = [s for s in sm.list_sessions(issue_id=issue.id) if s.model.status in ["running", "pending"]]
+                    if not active:
+                        # Check semaphore before spawning (including cooldown check)
+                        if not self.semaphore_manager.can_acquire("Engineer", issue_id=issue.id):
+                            logger.warning(f"Cannot spawn Engineer for {issue.id}: concurrency limit or cooldown active")
+                            continue
+                            
+                        logger.info(f"Handover trigger: Spawning Engineer for {issue.id}")
+                        self.spawn_engineer(sm, issue)
+        except Exception as e:
+            logger.error(f"Error in Handover trigger: {e}")
+
+    def spawn_engineer(self, sm: SessionManager, issue):
+        role = RoleTemplate(
+            name="Engineer",
+            description="Software Engineer",
+            trigger="handover",
+            goal=f"Implement feature: {issue.title}",
+            system_prompt="You are a Software Engineer. Read the issue and implement requirements.",
+            engine="gemini"
+        )
+        session = sm.create_session(issue_id=issue.id, role=role)
+        
+        # Acquire semaphore slot
+        self.semaphore_manager.acquire(session.model.id, "Engineer")
+        
+        try:
+            session.start()
+        except Exception as e:
+            # Release slot on spawn failure
+            self.semaphore_manager.release(session.model.id)
+            self.semaphore_manager.record_failure(issue.id, session.model.id)
+            logger.error(f"Failed to start Engineer session for {issue.id}: {e}")
+            raise
+
+    def spawn_architect(self, sm: SessionManager, project_context):
+        # Create Architect Session
+        role = RoleTemplate(
+            name="Architect",
+            description="System Architect",
+            trigger="memo.accumulation",
+            goal="Process memo inbox and create issues.",
+            system_prompt="You are the Architect. Process the Memo inbox.",
+            engine="gemini" # Default or from config?
+        )
+        session = sm.create_session(issue_id="architecture-review", role=role)
+        
+        # Acquire semaphore slot
+        self.semaphore_manager.acquire(session.model.id, "Architect")
+        
+        try:
+            session.start()
+        except Exception as e:
+            # Release slot on spawn failure
+            self.semaphore_manager.release(session.model.id)
+            logger.error(f"Failed to start Architect session: {e}")
+            raise
+
+    def handle_completion(self, session: RuntimeSession, sm: SessionManager):
+        # Chained Execution: Engineer -> Reviewer
+        if session.model.role_name == "Engineer":
+            # Check semaphore before spawning Reviewer
+            if not self.semaphore_manager.can_acquire("Reviewer", issue_id=session.model.issue_id):
+                logger.warning(
+                    f"Cannot spawn Reviewer for {session.model.issue_id}: "
+                    f"concurrency limit reached. Review will be deferred."
+                )
+                return
+                
+            logger.info(f"Engineer finished for {session.model.issue_id}. Spawning Reviewer.")
+            reviewer_role = RoleTemplate(
+                name="Reviewer",
+                description="Code Reviewer",
+                trigger="engineer.completion",
+                goal=f"Review work on {session.model.issue_id}",
+                system_prompt="You are a Code Reviewer. Review the code changes.",
+                engine="gemini"
+            )
+            rs = sm.create_session(issue_id=session.model.issue_id, role=reviewer_role)
+            
+            # Acquire semaphore slot
+            self.semaphore_manager.acquire(rs.model.id, "Reviewer")
+            
+            try:
+                rs.start()
+            except Exception as e:
+                # Release slot on spawn failure
+                self.semaphore_manager.release(rs.model.id)
+                logger.error(f"Failed to start Reviewer session for {session.model.issue_id}: {e}")
+                raise

monoco/daemon/services.py

@@ -2,6 +2,8 @@ import logging
 from typing import List, Optional, Dict, Any
 from asyncio import Queue
 from pathlib import Path
+from datetime import datetime, timedelta
+from threading import Lock
 
 import json
 
@@ -67,6 +69,189 @@ class ProjectContext:
         self.monitor.stop()
 
 
+class SemaphoreManager:
+    """
+    Manages concurrency limits for agent sessions using role-based semaphores.
+    Prevents fork bomb by limiting concurrent agents per role and globally.
+    """
+
+    def __init__(self, config: Optional[Any] = None):
+        """
+        Initialize the SemaphoreManager.
+        
+        Args:
+            config: AgentConcurrencyConfig or dict with concurrency settings
+        """
+        self._lock = Lock()
+        self._active_sessions: Dict[str, str] = {}  # session_id -> role_name
+        self._role_counts: Dict[str, int] = {}  # role_name -> count
+        self._failure_registry: Dict[str, datetime] = {}  # issue_id -> last_failure_time
+        
+        # Default conservative limits
+        self._global_max = 3
+        self._role_limits: Dict[str, int] = {
+            "Engineer": 1,
+            "Architect": 1,
+            "Reviewer": 1,
+            "Planner": 1,
+        }
+        self._failure_cooldown_seconds = 60
+        
+        # Apply config if provided
+        if config:
+            self._apply_config(config)
+    
+    def _apply_config(self, config: Any) -> None:
+        """Apply configuration settings."""
+        # Handle both dict and Pydantic model
+        if hasattr(config, 'global_max'):
+            self._global_max = config.global_max
+        if hasattr(config, 'failure_cooldown_seconds'):
+            self._failure_cooldown_seconds = config.failure_cooldown_seconds
+        
+        # Role-specific limits
+        for role in ["Engineer", "Architect", "Reviewer", "Planner"]:
+            if hasattr(config, role.lower()):
+                self._role_limits[role] = getattr(config, role.lower())
+    
+    def can_acquire(self, role_name: str, issue_id: Optional[str] = None) -> bool:
+        """
+        Check if a new session can be acquired for the given role.
+        
+        Args:
+            role_name: The role to check (e.g., "Engineer", "Architect")
+            issue_id: Optional issue ID to check for failure cooldown
+            
+        Returns:
+            True if the session can be started, False otherwise
+        """
+        with self._lock:
+            # Check global limit
+            total_active = len(self._active_sessions)
+            if total_active >= self._global_max:
+                logger.warning(
+                    f"Global concurrency limit reached ({self._global_max}). "
+                    f"Cannot spawn {role_name}."
+                )
+                return False
+            
+            # Check role-specific limit
+            role_count = self._role_counts.get(role_name, 0)
+            role_limit = self._role_limits.get(role_name, 1)
+            if role_count >= role_limit:
+                logger.warning(
+                    f"Role concurrency limit reached for {role_name} "
+                    f"({role_count}/{role_limit})."
+                )
+                return False
+            
+            # Check failure cooldown for this issue
+            if issue_id and issue_id in self._failure_registry:
+                last_failure = self._failure_registry[issue_id]
+                cooldown = timedelta(seconds=self._failure_cooldown_seconds)
+                if datetime.now() - last_failure < cooldown:
+                    remaining = cooldown - (datetime.now() - last_failure)
+                    logger.warning(
+                        f"Issue {issue_id} is in cooldown period. "
+                        f"Remaining: {remaining.seconds}s. Skipping spawn."
+                    )
+                    return False
+            
+            return True
+    
+    def acquire(self, session_id: str, role_name: str) -> bool:
+        """
+        Acquire a slot for a new session.
+        
+        Args:
+            session_id: Unique identifier for the session
+            role_name: The role of the session
+            
+        Returns:
+            True if acquired successfully, False otherwise
+        """
+        with self._lock:
+            if session_id in self._active_sessions:
+                logger.warning(f"Session {session_id} already tracked")
+                return True
+            
+            self._active_sessions[session_id] = role_name
+            self._role_counts[role_name] = self._role_counts.get(role_name, 0) + 1
+            logger.info(
+                f"Acquired slot for {role_name} session {session_id}. "
+                f"Global: {len(self._active_sessions)}/{self._global_max}, "
+                f"Role: {self._role_counts[role_name]}/{self._role_limits.get(role_name, 1)}"
+            )
+            return True
+    
+    def release(self, session_id: str) -> None:
+        """
+        Release a slot when a session ends.
+        
+        Args:
+            session_id: The session ID to release
+        """
+        with self._lock:
+            if session_id not in self._active_sessions:
+                return
+            
+            role_name = self._active_sessions.pop(session_id)
+            self._role_counts[role_name] = max(0, self._role_counts.get(role_name, 0) - 1)
+            logger.info(
+                f"Released slot for {role_name} session {session_id}. "
+                f"Global: {len(self._active_sessions)}/{self._global_max}"
+            )
+    
+    def record_failure(self, issue_id: str, session_id: Optional[str] = None) -> None:
+        """
+        Record a failure for cooldown purposes.
+        
+        Args:
+            issue_id: The issue that failed
+            session_id: Optional session ID to release
+        """
+        with self._lock:
+            self._failure_registry[issue_id] = datetime.now()
+            logger.warning(
+                f"Recorded failure for issue {issue_id}. "
+                f"Cooldown: {self._failure_cooldown_seconds}s"
+            )
+        
+        # Release the slot if session_id provided
+        if session_id:
+            self.release(session_id)
+    
+    def get_status(self) -> Dict[str, Any]:
+        """
+        Get current semaphore status for monitoring.
+        
+        Returns:
+            Dict with current counts and limits
+        """
+        with self._lock:
+            return {
+                "global": {
+                    "active": len(self._active_sessions),
+                    "limit": self._global_max,
+                },
+                "roles": {
+                    role: {
+                        "active": self._role_counts.get(role, 0),
+                        "limit": limit,
+                    }
+                    for role, limit in self._role_limits.items()
+                },
+                "cooldown_issues": len(self._failure_registry),
+            }
+    
+    def clear_failure(self, issue_id: str) -> None:
+        """Clear failure record for an issue (e.g., after successful completion)."""
+        with self._lock:
+            if issue_id in self._failure_registry:
+                del self._failure_registry[issue_id]
+                logger.info(f"Cleared failure record for issue {issue_id}")
+
+
 class ProjectManager:
     """
     Discovers and manages multiple Monoco projects within a workspace.

monoco/daemon/triggers.py

@@ -0,0 +1,55 @@
+from typing import TYPE_CHECKING, Optional, List, Any
+from pathlib import Path
+from monoco.features.memo.core import load_memos
+from monoco.features.issue.models import IssueMetadata, IssueStatus, IssueStage
+
+if TYPE_CHECKING:
+    from monoco.features.issue.models import IssueMetadata
+
+class TriggerPolicy:
+    """
+    Base class for trigger policies.
+    """
+    def evaluate(self, context: dict) -> bool:
+        raise NotImplementedError
+
+class MemoAccumulationPolicy(TriggerPolicy):
+    """
+    Trigger when pending memos exceed a threshold.
+    """
+    def __init__(self, count_threshold: int = 5):
+        self.count_threshold = count_threshold
+
+    def evaluate(self, context: dict) -> bool:
+        issues_root = context.get("issues_root")
+        if not issues_root:
+            return False
+        
+        if isinstance(issues_root, str):
+            issues_root = Path(issues_root)
+            
+        try:
+            memos = load_memos(issues_root)
+            pending_memos = [m for m in memos if m.status == "pending"]
+            return len(pending_memos) >= self.count_threshold
+        except Exception as e:
+            print(f"Error evaluating MemoAccumulationPolicy: {e}")
+            return False
+
+class HandoverPolicy(TriggerPolicy):
+    """
+    Trigger when an issue enters a specific state (e.g. Open/Doing for Engineer).
+    """
+    def __init__(self, target_status: IssueStatus, target_stage: IssueStage):
+        self.target_status = target_status
+        self.target_stage = target_stage
+
+    def evaluate(self, context: dict) -> bool:
+        issue: Optional[IssueMetadata] = context.get("issue")
+        if not issue:
+            return False
+            
+        return (
+            issue.status == self.target_status 
+            and issue.stage == self.target_stage
+        )