emdash-core 0.1.33__py3-none-any.whl → 0.1.37__py3-none-any.whl

emdash_core/agent/prompts/main_agent.py

@@ -37,12 +37,17 @@ def build_system_prompt(toolkit) -> str:
         Complete system prompt string
     """
     tools_section = build_tools_section(toolkit)
+    agents_section = build_agents_section(toolkit)
     skills_section = build_skills_section()
     rules_section = build_rules_section()
 
     # Main agent always uses the same prompt - it orchestrates and delegates
     prompt = BASE_SYSTEM_PROMPT.format(tools_section=tools_section)
 
+    # Add agents section so main agent knows what agents are available
+    if agents_section:
+        prompt += "\n" + agents_section
+
     # Add rules section if there are rules defined
     if rules_section:
         prompt += "\n" + rules_section
@@ -80,6 +85,36 @@ def build_skills_section() -> str:
     return registry.get_skills_for_prompt()
 
 
+def build_agents_section(toolkit) -> str:
+    """Build the agents section describing available sub-agents.
+
+    Args:
+        toolkit: The agent toolkit (to access repo_root)
+
+    Returns:
+        Formatted string with agent descriptions, or empty string if none
+    """
+    from ..toolkits import get_agents_with_descriptions
+
+    repo_root = getattr(toolkit, '_repo_root', None)
+    agents = get_agents_with_descriptions(repo_root)
+
+    if not agents:
+        return ""
+
+    lines = [
+        "## Available Agents",
+        "",
+        "Use the `task` tool to delegate work to these specialized agents:",
+        "",
+    ]
+
+    for agent in agents:
+        lines.append(f"- **{agent['name']}**: {agent['description']}")
+
+    return "\n".join(lines)
+
+
 def build_tools_section(toolkit) -> str:
     """Build the tools section of the system prompt from registered tools.
 

emdash_core/agent/prompts/subagents.py

@@ -178,12 +178,19 @@ SUBAGENT_PROMPTS = {
     "Research": RESEARCH_PROMPT,
 }
 
+# Built-in agent descriptions (for main agent's system prompt)
+BUILTIN_AGENTS = {
+    "Explore": "Fast codebase exploration - searches files, reads code, finds patterns",
+    "Plan": "Designs implementation plans - analyzes architecture, writes to .emdash/plans/",
+}
+
 
-def get_subagent_prompt(subagent_type: str) -> str:
+def get_subagent_prompt(subagent_type: str, repo_root=None) -> str:
     """Get the system prompt for a sub-agent type.
 
     Args:
-        subagent_type: Type of agent (e.g., "Explore", "Plan")
+        subagent_type: Type of agent (e.g., "Explore", "Plan", or custom agent name)
+        repo_root: Repository root for finding custom agents
 
     Returns:
         System prompt string
@@ -191,9 +198,18 @@ def get_subagent_prompt(subagent_type: str) -> str:
     Raises:
         ValueError: If agent type is not known
     """
-    if subagent_type not in SUBAGENT_PROMPTS:
-        available = list(SUBAGENT_PROMPTS.keys())
-        raise ValueError(
-            f"Unknown agent type: {subagent_type}. Available: {available}"
-        )
-    return SUBAGENT_PROMPTS[subagent_type]
+    # Check built-in agents first
+    if subagent_type in SUBAGENT_PROMPTS:
+        return SUBAGENT_PROMPTS[subagent_type]
+
+    # Check custom agents
+    from ..toolkits import get_custom_agent
+    custom_agent = get_custom_agent(subagent_type, repo_root)
+    if custom_agent:
+        return custom_agent.system_prompt
+
+    # Not found
+    available = list(SUBAGENT_PROMPTS.keys())
+    raise ValueError(
+        f"Unknown agent type: {subagent_type}. Available: {available}"
+    )

emdash_core/agent/prompts/workflow.py

@@ -104,6 +104,12 @@ glob("**/pages/**/*.astro")
 **Plan agent**: Implementation tasks that modify code
 - New features, refactoring, architectural changes
 - NOT for research/reading tasks
+
+**Custom agents** (from `.emdash/agents/*.md`):
+- User-defined specialized agents with custom system prompts
+- Spawned via `task(subagent_type="<agent-name>", prompt="...")`
+- Use the same tools as Explore agent (read-only by default)
+- Examples: security-audit, api-review, test-generator
 """
 
 # Exploration strategy for code navigation
@@ -311,40 +317,48 @@ SIZING_GUIDELINES = """
 
 # Todo list usage guidance
 TODO_LIST_GUIDANCE = """
-## Todo List Usage
+## Todo List Usage - USE PROACTIVELY
 
-You have access to `write_todo` and `update_todo_list` tools. Use them strategically - not for every task.
+You have access to `write_todo` and `update_todo_list` tools. **Use them frequently** to track progress and give the user visibility into what you're doing.
 
-### When to USE the todo list:
-- **3+ distinct steps** needed to complete the task
+### ALWAYS use the todo list when:
+- **2+ distinct steps** needed to complete the task
 - **Multiple files** need to be changed
 - **User gives a list** of tasks (numbered or comma-separated)
-- **Complex feature** implementation with multiple pieces
-- **Need to track progress** across iterations or when task spans multiple tool calls
+- **Any implementation task** that isn't trivial
+- **Multi-step debugging** or investigation
+- **Before starting work** - plan out what you'll do
+
+### Benefits of using todos:
+- User can see your progress in real-time
+- Helps you stay organized on complex tasks
+- Creates a clear record of what was done
+- Prevents forgetting steps in multi-part tasks
 
-### When to SKIP the todo list:
-- **Single focused change** (one edit, one file)
-- **Trivial fixes** (typo, add a log statement)
-- **Research/informational questions** (just answer them)
-- **Task completes in 1-2 steps** (just do it)
+### Only SKIP the todo list for:
+- **Truly trivial fixes** (single typo, one-line change)
+- **Simple questions** that need only a text answer
+- **Reading a single file** when asked
 
 ### Examples:
 
 **Use todo list:**
-- "Implement user authentication with login, logout, and session management" → 3+ steps, multiple files
-- "Fix these 5 type errors" → list of tasks
-- "Add dark mode support across the app" → complex, multiple files
+- "Fix the login bug" → investigate, identify cause, fix, verify
+- "Add a new API endpoint" → create route, handler, types, tests
+- "Update the config" → read current, plan changes, update, verify
+- "Implement dark mode" → multiple files, multiple steps
 
 **Skip todo list:**
-- "Fix the typo in README" → single focused change
-- "Add tool_choice parameter to this function" → one edit
-- "What files handle routing?" → informational question
-- "Update the error message here" → trivial fix
+- "What does this function do?" → just read and answer
+- "Fix typo in line 5" → single trivial edit
 
 ### Usage pattern:
-1. Use `write_todo(title="...", reset=true)` to start fresh with first task
-2. Use `write_todo(title="...")` to add more tasks
-3. Use `update_todo_list(task_id="1", status="in_progress")` when starting a task
-4. Use `update_todo_list(task_id="1", status="completed")` when done
-5. Mark tasks complete IMMEDIATELY after finishing - don't batch completions
+1. **Start immediately**: Use `write_todo(title="...", reset=true)` as soon as you understand the task
+2. **Add all steps**: Use `write_todo(title="...")` to add each step you'll take
+3. **Mark in_progress**: Use `update_todo_list(task_id="1", status="in_progress")` BEFORE starting each task
+4. **Mark completed**: Use `update_todo_list(task_id="1", status="completed")` IMMEDIATELY after finishing
+5. **Never batch**: Mark each task complete right away, don't wait
+
+### When in doubt, USE THE TODO LIST
+It's better to over-track than under-track. The user appreciates seeing progress.
 """

emdash_core/agent/runner/agent_runner.py

@@ -107,6 +107,8 @@ class AgentRunner(PlanMixin):
         self._tools_used_this_run: set[str] = set()
         # Plan approval state (from PlanMixin)
         self._pending_plan: Optional[dict] = None
+        # Callback for autosave after each iteration (set by API layer)
+        self._on_iteration_callback: Optional[callable] = None
 
     def _get_default_plan_file_path(self) -> str:
         """Get the default plan file path based on repo root.
@@ -504,6 +506,9 @@ class AgentRunner(PlanMixin):
                         "content": result_json,
                     })
 
+                # Emit context frame after each iteration (for autosave and UI updates)
+                self._emit_context_frame(messages)
+
                 # If a clarification question was asked, pause and wait for user input
                 if needs_user_input:
                     log.debug("Pausing agent loop - waiting for user input")
@@ -644,6 +649,13 @@ DO NOT output more text. Use a tool NOW.""",
             total_output_tokens=self._total_output_tokens,
         )
 
+        # Call iteration callback for autosave if set
+        if self._on_iteration_callback and messages:
+            try:
+                self._on_iteration_callback(messages)
+            except Exception as e:
+                log.debug(f"Iteration callback failed: {e}")
+
     def chat(self, message: str, images: Optional[list] = None) -> str:
         """Continue a conversation with a new message.
 

emdash_core/agent/runner/context.py

@@ -4,6 +4,7 @@ This module contains functions for estimating, compacting, and managing
 conversation context during agent runs.
 """
 
+import os
 from typing import Optional, TYPE_CHECKING
 
 from ...utils.logger import log
@@ -299,7 +300,7 @@ def get_reranked_context(
         # Get exploration steps for context extraction
         steps = toolkit.get_exploration_steps()
         if not steps:
-            return {"item_count": 0, "items": []}
+            return {"item_count": 0, "items": [], "query": current_query, "debug": "no exploration steps"}
 
         # Use context service to extract context items from exploration
         service = ContextService(connection=toolkit.connection)
@@ -314,34 +315,52 @@ def get_reranked_context(
         # Get context items
         items = service.get_context_items(terminal_id)
         if not items:
-            return {"item_count": 0, "items": []}
+            return {"item_count": 0, "items": [], "query": current_query, "debug": f"no items from service ({len(steps)} steps)"}
+
+        # Get max tokens from env (default 2000)
+        max_tokens = int(os.getenv("CONTEXT_FRAME_MAX_TOKENS", "2000"))
 
         # Rerank by query relevance
         if current_query:
             items = rerank_context_items(
                 items,
                 current_query,
-                top_k=20,
+                top_k=50,  # Get more candidates, then filter by tokens
             )
 
-        # Convert to serializable format
+        # Convert to serializable format, limiting by token count
         result_items = []
-        for item in items[:20]:  # Limit to 20 items
-            result_items.append({
+        total_tokens = 0
+        for item in items:
+            item_dict = {
                 "name": item.qualified_name,
                 "type": item.entity_type,
                 "file": item.file_path,
                 "score": round(item.score, 3) if hasattr(item, 'score') else None,
-            })
+                "description": item.description[:200] if item.description else None,
+                "touch_count": item.touch_count,
+                "neighbors": item.neighbors[:5] if item.neighbors else [],
+            }
+            # Estimate tokens for this item (~4 chars per token)
+            item_chars = len(str(item_dict))
+            item_tokens = item_chars // 4
+
+            if total_tokens + item_tokens > max_tokens:
+                break
+
+            result_items.append(item_dict)
+            total_tokens += item_tokens
 
         return {
             "item_count": len(result_items),
             "items": result_items,
+            "query": current_query,
+            "total_tokens": total_tokens,
         }
 
     except Exception as e:
-        log.debug(f"Failed to get reranked context: {e}")
-        return {"item_count": 0, "items": []}
+        log.warning(f"Failed to get reranked context: {e}")
+        return {"item_count": 0, "items": [], "query": current_query, "debug": str(e)}
 
 
 def emit_context_frame(

emdash_core/agent/toolkits/__init__.py

@@ -2,10 +2,13 @@
 
 Provides specialized toolkits for different agent types.
 Each toolkit contains a curated set of tools appropriate for the agent's purpose.
+
+Custom agents from .emdash/agents/*.md are also supported and use the Explore toolkit
+by default (unless they specify different tools in their frontmatter).
 """
 
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, Type
+from typing import TYPE_CHECKING, Dict, Type, Optional
 
 if TYPE_CHECKING:
     from .base import BaseToolkit
@@ -20,45 +23,141 @@ TOOLKIT_REGISTRY: Dict[str, str] = {
 }
 
 
+def _get_custom_agents(repo_root: Optional[Path] = None) -> dict:
+    """Load custom agents from .emdash/agents/ directory.
+
+    Args:
+        repo_root: Repository root (defaults to cwd)
+
+    Returns:
+        Dict mapping agent name to CustomAgent
+    """
+    from ..agents import load_agents
+    from ...utils.logger import log
+
+    agents_dir = (repo_root or Path.cwd()) / ".emdash" / "agents"
+    log.debug(f"Loading custom agents from: {agents_dir} (exists={agents_dir.exists()})")
+    agents = load_agents(agents_dir)
+    log.debug(f"Loaded custom agents: {list(agents.keys())}")
+    return agents
+
+
 def get_toolkit(subagent_type: str, repo_root: Path) -> "BaseToolkit":
     """Get toolkit for agent type.
 
     Args:
-        subagent_type: Type of agent (e.g., "Explore", "Plan")
+        subagent_type: Type of agent (e.g., "Explore", "Plan", or custom agent name)
         repo_root: Root directory of the repository
 
     Returns:
         Toolkit instance
 
     Raises:
-        ValueError: If agent type is not registered
+        ValueError: If agent type is not registered or found
+    """
+    # Check built-in agents first
+    if subagent_type in TOOLKIT_REGISTRY:
+        import importlib
+        module_path, class_name = TOOLKIT_REGISTRY[subagent_type].rsplit(":", 1)
+        module = importlib.import_module(module_path)
+        toolkit_class = getattr(module, class_name)
+        return toolkit_class(repo_root)
+
+    # Check custom agents
+    custom_agents = _get_custom_agents(repo_root)
+    if subagent_type in custom_agents:
+        # Custom agents use Explore toolkit by default (read-only, safe)
+        # This gives them: glob, grep, read_file, list_files, semantic_search
+        # Plus any MCP servers defined in the agent's frontmatter
+        import importlib
+        from ...utils.logger import log
+
+        custom_agent = custom_agents[subagent_type]
+        module_path, class_name = TOOLKIT_REGISTRY["Explore"].rsplit(":", 1)
+        module = importlib.import_module(module_path)
+        toolkit_class = getattr(module, class_name)
+
+        # Pass MCP servers if defined
+        mcp_servers = custom_agent.mcp_servers if custom_agent.mcp_servers else None
+        if mcp_servers:
+            log.info(f"Custom agent '{subagent_type}' has {len(mcp_servers)} MCP servers")
+
+        return toolkit_class(repo_root, mcp_servers=mcp_servers)
+
+    # Not found
+    available = list_agent_types(repo_root)
+    raise ValueError(
+        f"Unknown agent type: {subagent_type}. Available: {available}"
+    )
+
+
+def list_agent_types(repo_root: Optional[Path] = None) -> list[str]:
+    """List all available agent types (built-in + custom).
+
+    Args:
+        repo_root: Repository root for finding custom agents
+
+    Returns:
+        List of agent type names
     """
-    if subagent_type not in TOOLKIT_REGISTRY:
-        available = list(TOOLKIT_REGISTRY.keys())
-        raise ValueError(
-            f"Unknown agent type: {subagent_type}. Available: {available}"
-        )
+    # Start with built-in agents
+    types = list(TOOLKIT_REGISTRY.keys())
 
-    # Import lazily to avoid circular imports
-    import importlib
+    # Add custom agents
+    custom_agents = _get_custom_agents(repo_root)
+    for name in custom_agents.keys():
+        if name not in types:
+            types.append(name)
 
-    module_path, class_name = TOOLKIT_REGISTRY[subagent_type].rsplit(":", 1)
-    module = importlib.import_module(module_path)
-    toolkit_class = getattr(module, class_name)
-    return toolkit_class(repo_root)
+    return types
 
 
-def list_agent_types() -> list[str]:
-    """List all available agent types.
+def get_agents_with_descriptions(repo_root: Optional[Path] = None) -> list[dict]:
+    """Get all agents with their names and descriptions.
+
+    Args:
+        repo_root: Repository root for finding custom agents
 
     Returns:
-        List of agent type names
+        List of dicts with 'name' and 'description' keys
+    """
+    from ..prompts.subagents import BUILTIN_AGENTS
+
+    agents = []
+
+    # Built-in agents
+    for name, description in BUILTIN_AGENTS.items():
+        agents.append({"name": name, "description": description})
+
+    # Custom agents
+    custom_agents = _get_custom_agents(repo_root)
+    for name, agent in custom_agents.items():
+        agents.append({
+            "name": name,
+            "description": agent.description or "Custom agent"
+        })
+
+    return agents
+
+
+def get_custom_agent(name: str, repo_root: Optional[Path] = None):
+    """Get a specific custom agent by name.
+
+    Args:
+        name: Agent name
+        repo_root: Repository root
+
+    Returns:
+        CustomAgent or None
     """
-    return list(TOOLKIT_REGISTRY.keys())
+    custom_agents = _get_custom_agents(repo_root)
+    return custom_agents.get(name)
 
 
 __all__ = [
     "get_toolkit",
     "list_agent_types",
+    "get_agents_with_descriptions",
+    "get_custom_agent",
     "TOOLKIT_REGISTRY",
 ]

emdash_core/agent/toolkits/base.py

@@ -2,10 +2,14 @@
 
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from ..tools.base import BaseTool, ToolResult
 
+if TYPE_CHECKING:
+    from ..agents import AgentMCPServerConfig
+    from ..mcp.manager import MCPServerManager
+
 
 class BaseToolkit(ABC):
     """Abstract base class for sub-agent toolkits.
@@ -15,20 +19,30 @@ class BaseToolkit(ABC):
     - Registering appropriate tools
     - Providing OpenAI function schemas
     - Executing tools by name
+    - Managing per-agent MCP servers (optional)
     """
 
     # List of tool names this toolkit provides (for documentation)
     TOOLS: list[str] = []
 
-    def __init__(self, repo_root: Path):
+    def __init__(
+        self,
+        repo_root: Path,
+        mcp_servers: Optional[list["AgentMCPServerConfig"]] = None,
+    ):
         """Initialize the toolkit.
 
         Args:
             repo_root: Root directory of the repository
+            mcp_servers: Optional list of MCP server configurations for this agent
         """
         self.repo_root = repo_root.resolve()
         self._tools: dict[str, BaseTool] = {}
+        self._mcp_manager: Optional["MCPServerManager"] = None
+        self._mcp_servers_config = mcp_servers or []
+
         self._register_tools()
+        self._init_mcp_servers()
 
     @abstractmethod
     def _register_tools(self) -> None:
@@ -38,6 +52,77 @@ class BaseToolkit(ABC):
         """
         pass
 
+    def _init_mcp_servers(self) -> None:
+        """Initialize MCP servers for this agent if configured.
+
+        Creates an MCPServerManager with the agent's MCP server configs
+        and registers the tools from those servers. Only enabled servers
+        are started.
+        """
+        if not self._mcp_servers_config:
+            return
+
+        # Filter to only enabled servers
+        enabled_servers = [s for s in self._mcp_servers_config if s.enabled]
+        if not enabled_servers:
+            return
+
+        from ..mcp.config import MCPServerConfig, MCPConfigFile
+        from ..mcp.manager import MCPServerManager
+        from ..mcp.tool_factory import create_tools_from_mcp
+        from ...utils.logger import log
+
+        log.info(f"Initializing {len(enabled_servers)} MCP servers for agent")
+
+        # Create a temporary config file object with our servers
+        config = MCPConfigFile()
+        for server_cfg in enabled_servers:
+            mcp_config = MCPServerConfig(
+                name=server_cfg.name,
+                command=server_cfg.command,
+                args=server_cfg.args,
+                env=server_cfg.env,
+                enabled=True,
+                timeout=server_cfg.timeout,
+            )
+            config.add_server(mcp_config)
+
+        # Create manager with in-memory config (not from file)
+        self._mcp_manager = MCPServerManager(repo_root=self.repo_root)
+        self._mcp_manager._config = config  # Inject our config directly
+
+        # Start all servers and register tools
+        try:
+            started = self._mcp_manager.start_all_enabled()
+            log.info(f"Started MCP servers for agent: {started}")
+
+            # Create tool wrappers and register them
+            mcp_tools = create_tools_from_mcp(self._mcp_manager)
+            for tool in mcp_tools:
+                self.register_tool(tool)
+                log.debug(f"Registered MCP tool: {tool.name}")
+
+        except Exception as e:
+            log.warning(f"Failed to initialize MCP servers for agent: {e}")
+
+    def shutdown(self) -> None:
+        """Shutdown the toolkit and cleanup resources.
+
+        Stops any running MCP servers.
+        """
+        if self._mcp_manager:
+            from ...utils.logger import log
+            log.info("Shutting down agent MCP servers")
+            self._mcp_manager.shutdown_all()
+            self._mcp_manager = None
+
+    def __del__(self):
+        """Cleanup on garbage collection."""
+        try:
+            self.shutdown()
+        except Exception:
+            pass
+
     def register_tool(self, tool: BaseTool) -> None:
         """Register a tool.
 

emdash_core/agent/toolkits/explore.py

@@ -1,12 +1,16 @@
 """Explorer toolkit - read-only tools for fast codebase exploration."""
 
 from pathlib import Path
+from typing import TYPE_CHECKING, Optional
 
 from .base import BaseToolkit
 from ..tools.coding import ReadFileTool, ListFilesTool
 from ..tools.search import SemanticSearchTool, GrepTool, GlobTool
 from ...utils.logger import log
 
+if TYPE_CHECKING:
+    from ..agents import AgentMCPServerConfig
+
 
 class ExploreToolkit(BaseToolkit):
     """Read-only toolkit for fast codebase exploration.
@@ -16,6 +20,7 @@ class ExploreToolkit(BaseToolkit):
     - Listing directory contents
     - Searching with patterns (grep, glob)
     - Semantic code search
+    - MCP server tools (if configured)
 
     All tools are read-only - no file modifications allowed.
     """
@@ -28,6 +33,19 @@ class ExploreToolkit(BaseToolkit):
         "semantic_search",
     ]
 
+    def __init__(
+        self,
+        repo_root: Path,
+        mcp_servers: Optional[list["AgentMCPServerConfig"]] = None,
+    ):
+        """Initialize the explore toolkit.
+
+        Args:
+            repo_root: Root directory of the repository
+            mcp_servers: Optional MCP server configurations for this agent
+        """
+        super().__init__(repo_root, mcp_servers=mcp_servers)
+
     def _register_tools(self) -> None:
         """Register read-only exploration tools."""
         # File reading