emdash-core 0.1.33__py3-none-any.whl → 0.1.60__py3-none-any.whl

emdash_core/agent/runner/agent_runner.py

@@ -11,6 +11,7 @@ from typing import Any, Optional
 
 from ...utils.logger import log
 from ...core.exceptions import ContextLengthError
+from ...core.config import get_config
 from ..toolkit import AgentToolkit
 from ..events import AgentEventEmitter, NullEmitter
 from ..providers import get_provider
@@ -30,8 +31,10 @@ from .context import (
     get_context_breakdown,
     maybe_compact_context,
     emit_context_frame,
+    get_reranked_context,
 )
 from .plan import PlanMixin
+from ..background import BackgroundTaskManager
 
 
 class AgentRunner(PlanMixin):
@@ -107,6 +110,14 @@ class AgentRunner(PlanMixin):
         self._tools_used_this_run: set[str] = set()
         # Plan approval state (from PlanMixin)
         self._pending_plan: Optional[dict] = None
+        # Callback for autosave after each iteration (set by API layer)
+        self._on_iteration_callback: Optional[callable] = None
+        # Context frame injection flag
+        self._inject_context_frame = os.getenv("EMDASH_INJECT_CONTEXT_FRAME", "").lower() in ("1", "true", "yes")
+        # Persistent thread pool executor for parallel tool execution
+        config = get_config()
+        self._tool_parallel_workers = config.agent.tool_parallel_workers
+        self._tool_executor: Optional[ThreadPoolExecutor] = None
 
     def _get_default_plan_file_path(self) -> str:
         """Get the default plan file path based on repo root.
@@ -141,9 +152,101 @@ class AgentRunner(PlanMixin):
         task_list = "\n".join(lines)
         return f"<todo-state>\n{header}\n{task_list}\n</todo-state>"
 
+    def _check_background_notifications(self) -> list[str]:
+        """Check for completed background tasks and format notifications.
+
+        Returns:
+            List of notification messages to inject into context
+        """
+        try:
+            manager = BackgroundTaskManager.get_instance()
+            completed_tasks = manager.get_pending_notifications()
+
+            notifications = []
+            for task in completed_tasks:
+                msg = manager.format_notification(task)
+                notifications.append(msg)
+                log.info(f"Background task {task.task_id} notification ready")
+
+            return notifications
+        except Exception as e:
+            log.warning(f"Failed to check background notifications: {e}")
+            return []
+
+    def _format_context_reminder(self) -> str:
+        """Format reranked context items as XML reminder for injection.
+
+        Only called when EMDASH_INJECT_CONTEXT_FRAME is enabled.
+
+        Returns:
+            Formatted context reminder string, or empty if no context
+        """
+        if not self._current_query:
+            return ""
+
+        reading = get_reranked_context(self.toolkit, self._current_query)
+        items = reading.get("items", [])
+
+        if not items:
+            return ""
+
+        lines = [
+            "<context-frame>",
+            f"Relevant context for query: {self._current_query[:100]}",
+            f"Found {len(items)} relevant items (ranked by relevance score):",
+            "",
+        ]
+
+        for item in items[:15]:  # Top 15 items
+            name = item.get("name", "?")
+            item_type = item.get("type", "?")
+            score = item.get("score")
+            file_path = item.get("file", "")
+            description = item.get("description", "")
+
+            score_str = f" (score: {score:.3f})" if score is not None else ""
+            file_str = f" in {file_path}" if file_path else ""
+
+            lines.append(f"  - [{item_type}] {name}{score_str}{file_str}")
+            if description:
+                lines.append(f"    {description[:150]}")
+
+        lines.append("</context-frame>")
+        return "\n".join(lines)
+
+    def _get_tool_executor(self) -> ThreadPoolExecutor:
+        """Get the persistent thread pool executor, creating it if needed.
+
+        Uses lazy initialization to avoid creating threads until actually needed.
+        """
+        if self._tool_executor is None:
+            self._tool_executor = ThreadPoolExecutor(
+                max_workers=self._tool_parallel_workers,
+                thread_name_prefix="tool-exec-"
+            )
+        return self._tool_executor
+
+    def close(self) -> None:
+        """Clean up resources, including the thread pool executor."""
+        if self._tool_executor is not None:
+            self._tool_executor.shutdown(wait=False)
+            self._tool_executor = None
+
+    def __enter__(self):
+        """Support context manager protocol."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Clean up on exit from context manager."""
+        self.close()
+        return False
+
     def _execute_tools_parallel(self, parsed_calls: list) -> list:
         """Execute multiple tool calls in parallel using a thread pool.
 
+        Uses a persistent thread pool executor for better performance by avoiding
+        thread creation/destruction overhead on each batch of tool calls.
+
         Args:
             parsed_calls: List of (tool_call, args) tuples
 
@@ -164,14 +267,14 @@ class AgentRunner(PlanMixin):
                 from ..tools.base import ToolResult
                 return (tool_call, args, ToolResult.error_result(str(e)))
 
-        # Execute in parallel with up to 3 workers
+        # Execute in parallel using persistent executor
+        executor = self._get_tool_executor()
         results: list = [None] * len(parsed_calls)
-        with ThreadPoolExecutor(max_workers=3) as executor:
-            futures = {executor.submit(execute_one, item): i for i, item in enumerate(parsed_calls)}
-            # Collect results maintaining order
-            for future in as_completed(futures):
-                idx = futures[future]
-                results[idx] = future.result()
+        futures = {executor.submit(execute_one, item): i for i, item in enumerate(parsed_calls)}
+        # Collect results maintaining order
+        for future in as_completed(futures):
+            idx = futures[future]
+            results[idx] = future.result()
 
         # Emit tool result events for all calls
         for tool_call, args, result in results:
@@ -207,24 +310,27 @@ class AgentRunner(PlanMixin):
         from ..tools.modes import ModeState
         ModeState.get_instance().reset_cycle()
 
-        # Build user message
+        # Build user message content
         if context:
-            user_message = {
-                "role": "user",
-                "content": f"Context:\n{context}\n\nQuestion: {query}",
-            }
+            text_content = f"Context:\n{context}\n\nQuestion: {query}"
         else:
-            user_message = {
-                "role": "user",
-                "content": query,
-            }
+            text_content = query
+
+        # Format content with images if provided
+        if images:
+            content = self.provider.format_content_with_images(text_content, images)
+        else:
+            content = text_content
+
+        user_message = {
+            "role": "user",
+            "content": content,
+        }
 
         # Save user message to history BEFORE running (so it's preserved even if interrupted)
         self._messages.append(user_message)
         messages = list(self._messages)  # Copy for the loop
 
-        # TODO: Handle images if provided
-
         # Get tool schemas
         tools = self.toolkit.get_all_schemas()
 
@@ -260,6 +366,16 @@ class AgentRunner(PlanMixin):
         max_retries = 3
 
         for iteration in range(self.max_iterations):
+            # Check for completed background tasks and inject notifications
+            bg_notifications = self._check_background_notifications()
+            for notification in bg_notifications:
+                messages.append({
+                    "role": "user",
+                    "content": notification,
+                })
+                # Emit event so UI can show notification
+                self.emitter.emit_assistant_text(f"[Background task completed - see notification]")
+
             # When approaching max iterations, ask agent to wrap up
             if iteration == self.max_iterations - 2:
                 messages.append({
@@ -504,6 +620,15 @@ class AgentRunner(PlanMixin):
                         "content": result_json,
                     })
 
+                # Inject context frame reminder if enabled (append to last tool result)
+                if self._inject_context_frame and messages and messages[-1].get("role") == "tool":
+                    context_reminder = self._format_context_reminder()
+                    if context_reminder:
+                        messages[-1]["content"] += f"\n\n{context_reminder}"
+
+                # Emit context frame after each iteration (for autosave and UI updates)
+                self._emit_context_frame(messages)
+
                 # If a clarification question was asked, pause and wait for user input
                 if needs_user_input:
                     log.debug("Pausing agent loop - waiting for user input")
@@ -644,6 +769,13 @@ DO NOT output more text. Use a tool NOW.""",
             total_output_tokens=self._total_output_tokens,
         )
 
+        # Call iteration callback for autosave if set
+        if self._on_iteration_callback and messages:
+            try:
+                self._on_iteration_callback(messages)
+            except Exception as e:
+                log.debug(f"Iteration callback failed: {e}")
+
     def chat(self, message: str, images: Optional[list] = None) -> str:
         """Continue a conversation with a new message.
 
@@ -664,10 +796,16 @@ DO NOT output more text. Use a tool NOW.""",
         # Store query for reranking context frame
         self._current_query = message
 
+        # Format content with images if provided
+        if images:
+            content = self.provider.format_content_with_images(message, images)
+        else:
+            content = message
+
         # Add new user message to history
         self._messages.append({
             "role": "user",
-            "content": message,
+            "content": content,
         })
 
         # Get tool schemas

emdash_core/agent/runner/context.py

@@ -4,6 +4,7 @@ This module contains functions for estimating, compacting, and managing
 conversation context during agent runs.
 """
 
+import os
 from typing import Optional, TYPE_CHECKING
 
 from ...utils.logger import log
@@ -299,7 +300,7 @@ def get_reranked_context(
         # Get exploration steps for context extraction
         steps = toolkit.get_exploration_steps()
         if not steps:
-            return {"item_count": 0, "items": []}
+            return {"item_count": 0, "items": [], "query": current_query, "debug": "no exploration steps"}
 
         # Use context service to extract context items from exploration
         service = ContextService(connection=toolkit.connection)
@@ -314,34 +315,52 @@ def get_reranked_context(
         # Get context items
         items = service.get_context_items(terminal_id)
         if not items:
-            return {"item_count": 0, "items": []}
+            return {"item_count": 0, "items": [], "query": current_query, "debug": f"no items from service ({len(steps)} steps)"}
+
+        # Get max tokens from env (default 2000)
+        max_tokens = int(os.getenv("CONTEXT_FRAME_MAX_TOKENS", "2000"))
 
         # Rerank by query relevance
         if current_query:
             items = rerank_context_items(
                 items,
                 current_query,
-                top_k=20,
+                top_k=50,  # Get more candidates, then filter by tokens
             )
 
-        # Convert to serializable format
+        # Convert to serializable format, limiting by token count
         result_items = []
-        for item in items[:20]:  # Limit to 20 items
-            result_items.append({
+        total_tokens = 0
+        for item in items:
+            item_dict = {
                 "name": item.qualified_name,
                 "type": item.entity_type,
                 "file": item.file_path,
                 "score": round(item.score, 3) if hasattr(item, 'score') else None,
-            })
+                "description": item.description[:200] if item.description else None,
+                "touch_count": item.touch_count,
+                "neighbors": item.neighbors[:5] if item.neighbors else [],
+            }
+            # Estimate tokens for this item (~4 chars per token)
+            item_chars = len(str(item_dict))
+            item_tokens = item_chars // 4
+
+            if total_tokens + item_tokens > max_tokens:
+                break
+
+            result_items.append(item_dict)
+            total_tokens += item_tokens
 
         return {
             "item_count": len(result_items),
             "items": result_items,
+            "query": current_query,
+            "total_tokens": total_tokens,
         }
 
     except Exception as e:
-        log.debug(f"Failed to get reranked context: {e}")
-        return {"item_count": 0, "items": []}
+        log.warning(f"Failed to get reranked context: {e}")
+        return {"item_count": 0, "items": [], "query": current_query, "debug": str(e)}
 
 
 def emit_context_frame(

emdash_core/agent/runner/sdk_runner.py

@@ -184,11 +184,12 @@ class SDKAgentRunner:
             from ..events import EventType
             self.emitter.emit(getattr(EventType, event_type), data)
 
-    async def run(self, prompt: str) -> AsyncIterator[dict]:
+    async def run(self, prompt: str, images: list = None) -> AsyncIterator[dict]:
         """Execute agent with SDK.
 
         Args:
             prompt: User prompt/task
+            images: Optional list of image dicts with 'data' (bytes) and 'format' keys
 
         Yields:
             Event dicts for UI streaming
@@ -201,6 +202,7 @@ class SDKAgentRunner:
             ToolResultBlock,
             ResultMessage,
         )
+        import base64
 
         options = self._get_options()
 
@@ -210,9 +212,34 @@ class SDKAgentRunner:
             "agent_name": "Emdash Code (SDK)",
         })
 
+        # Format prompt with images if provided
+        if images:
+            # Build content blocks for Claude SDK
+            content_blocks = []
+            for img in images:
+                img_data = img.get("data")
+                img_format = img.get("format", "png")
+                if isinstance(img_data, bytes):
+                    encoded = base64.b64encode(img_data).decode("utf-8")
+                else:
+                    encoded = img_data  # Already base64 encoded
+                content_blocks.append({
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": f"image/{img_format}",
+                        "data": encoded,
+                    }
+                })
+            content_blocks.append({"type": "text", "text": prompt})
+            query_content = content_blocks
+            log.info(f"SDK agent: sending {len(images)} images with prompt")
+        else:
+            query_content = prompt
+
         try:
             async with ClaudeSDKClient(options=options) as client:
-                await client.query(prompt)
+                await client.query(query_content)
 
                 async for message in client.receive_response():
                     # Process message and yield events

emdash_core/agent/skills.py

@@ -16,6 +16,9 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Optional
 
+import os
+import stat
+
 from ..utils.logger import log
 
 
@@ -24,6 +27,66 @@ def _get_builtin_skills_dir() -> Path:
     return Path(__file__).parent.parent / "skills"
 
 
+def _discover_scripts(skill_dir: Path) -> list[Path]:
+    """Discover executable scripts in a skill directory.
+
+    Scripts are self-contained bash executables that can be run by the agent
+    to perform specific actions. They must be either:
+    - Files with .sh extension
+    - Files with executable permission and a shebang (#!/bin/bash, #!/usr/bin/env bash, etc.)
+
+    Args:
+        skill_dir: Path to the skill directory
+
+    Returns:
+        List of paths to executable scripts
+    """
+    scripts = []
+
+    if not skill_dir.exists() or not skill_dir.is_dir():
+        return scripts
+
+    # Files to skip (not scripts)
+    skip_files = {"SKILL.md", "skill.md", "README.md", "readme.md"}
+
+    for file_path in skill_dir.iterdir():
+        if not file_path.is_file():
+            continue
+
+        if file_path.name in skip_files:
+            continue
+
+        # Check if it's a .sh file
+        is_shell_script = file_path.suffix == ".sh"
+
+        # Check if it has a shebang
+        has_shebang = False
+        try:
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                first_line = f.readline().strip()
+                if first_line.startswith("#!"):
+                    # Check for bash/sh shebang
+                    if any(shell in first_line for shell in ["bash", "/sh", "python", "node", "ruby", "perl"]):
+                        has_shebang = True
+        except (OSError, IOError):
+            continue
+
+        if is_shell_script or has_shebang:
+            # Ensure the file is executable
+            try:
+                current_mode = file_path.stat().st_mode
+                if not (current_mode & stat.S_IXUSR):
+                    # Make it executable for the user
+                    os.chmod(file_path, current_mode | stat.S_IXUSR)
+                    log.debug(f"Made script executable: {file_path}")
+            except OSError as e:
+                log.warning(f"Could not make script executable: {file_path}: {e}")
+
+            scripts.append(file_path)
+
+    return sorted(scripts, key=lambda p: p.name)
+
+
 @dataclass
 class Skill:
     """A skill configuration loaded from SKILL.md.
@@ -35,6 +98,7 @@ class Skill:
         tools: List of tools this skill needs access to
         user_invocable: Whether skill can be invoked with /name
         file_path: Source file path
+        scripts: List of executable script paths in the skill directory
         _builtin: Whether this is a built-in skill bundled with emdash_core
     """
 
@@ -44,6 +108,7 @@ class Skill:
     tools: list[str] = field(default_factory=list)
     user_invocable: bool = False
     file_path: Optional[Path] = None
+    scripts: list[Path] = field(default_factory=list)
     _builtin: bool = False
 
 
@@ -166,6 +231,10 @@ class SkillRegistry:
                 skill = _parse_skill_file(skill_file, skill_dir.name)
                 if skill:
                     skill._builtin = is_builtin  # Mark as built-in or user-defined
+                    # Discover scripts in the skill directory
+                    skill.scripts = _discover_scripts(skill_dir)
+                    if skill.scripts:
+                        log.debug(f"Found {len(skill.scripts)} scripts in skill: {skill.name}")
                     skills[skill.name] = skill
                     self._skills[skill.name] = skill
                     source = "built-in" if is_builtin else "user"
@@ -224,12 +293,23 @@ class SkillRegistry:
 
         for skill in self._skills.values():
             invocable = " (user-invocable: /{})".format(skill.name) if skill.user_invocable else ""
-            lines.append(f"- **{skill.name}**: {skill.description}{invocable}")
+            scripts_note = f" [has {len(skill.scripts)} script(s)]" if skill.scripts else ""
+            lines.append(f"- **{skill.name}**: {skill.description}{invocable}{scripts_note}")
 
         lines.append("")
         lines.append("To activate a skill, use the `skill` tool with the skill name.")
         lines.append("")
 
+        # Add note about skill scripts if any skill has scripts
+        has_scripts = any(skill.scripts for skill in self._skills.values())
+        if has_scripts:
+            lines.append("### Skill Scripts")
+            lines.append("")
+            lines.append("Some skills include executable scripts that can be run using the Bash tool.")
+            lines.append("When you invoke a skill with scripts, the script paths will be provided.")
+            lines.append("Scripts are self-contained and can be executed directly.")
+            lines.append("")
+
         return "\n".join(lines)
 
 

emdash_core/agent/toolkit.py

@@ -1,9 +1,10 @@
 """Main AgentToolkit class for LLM agent graph exploration."""
 
+import os
 from pathlib import Path
 from typing import Optional
 
-from ..graph.connection import KuzuConnection, get_connection
+from ..graph.connection import KuzuConnection, get_connection, KUZU_AVAILABLE
 from .tools.base import BaseTool, ToolResult, ToolCategory
 from .session import AgentSession
 from ..utils.logger import log
@@ -56,7 +57,20 @@ class AgentToolkit:
             save_spec_path: If provided, specs will be saved to this path.
             plan_file_path: Path to the plan file (only writable file in plan mode).
         """
-        self.connection = connection or get_connection()
+        # Handle connection - Kuzu is optional
+        if connection is not None:
+            self.connection = connection
+        elif KUZU_AVAILABLE:
+            try:
+                self.connection = get_connection()
+            except Exception as e:
+                log.warning(f"Failed to connect to Kuzu database: {e}")
+                log.warning("Semantic search will be disabled. Other tools will work normally.")
+                self.connection = None
+        else:
+            log.info("Kuzu not installed - semantic search disabled. Install with: pip install kuzu")
+            self.connection = None
+
         self.session = AgentSession() if enable_session else None
         self._tools: dict[str, BaseTool] = {}
         self._mcp_manager = None
@@ -104,8 +118,10 @@ class AgentToolkit:
         )
 
         # Register search tools
-        self.register_tool(SemanticSearchTool(self.connection))
-        # self.register_tool(TextSearchTool(self.connection))  # Disabled due to DB locking issues
+        # SemanticSearchTool requires Kuzu database
+        if self.connection is not None:
+            self.register_tool(SemanticSearchTool(self.connection))
+        # These tools work without database connection
         self.register_tool(GrepTool(self.connection))
         self.register_tool(GlobTool(self.connection))
         self.register_tool(WebTool(self.connection))
@@ -131,15 +147,21 @@ class AgentToolkit:
             # In code mode: full write access
             from .tools.coding import (
                 WriteToFileTool,
-                ApplyDiffTool,
                 DeleteFileTool,
                 ExecuteCommandTool,
             )
             self.register_tool(WriteToFileTool(self._repo_root, self.connection))
-            self.register_tool(ApplyDiffTool(self._repo_root, self.connection))
             self.register_tool(DeleteFileTool(self._repo_root, self.connection))
             self.register_tool(ExecuteCommandTool(self._repo_root, self.connection))
 
+            # Toggle between apply_diff (default) and edit_file based on env var
+            if os.getenv("EMDASH_ENABLE_APPLY_DIFF", "true").lower() in ("0", "false", "no"):
+                from .tools.coding import EditFileTool
+                self.register_tool(EditFileTool(self._repo_root, self.connection))
+            else:
+                from .tools.coding import ApplyDiffTool
+                self.register_tool(ApplyDiffTool(self._repo_root, self.connection))
+
         # Register sub-agent tools for spawning lightweight agents
         self._register_subagent_tools()
 
@@ -169,16 +191,22 @@ class AgentToolkit:
         log.debug(f"Registered {len(self._tools)} agent tools")
 
     def _register_subagent_tools(self) -> None:
-        """Register sub-agent tools for spawning lightweight agents.
-
-        These tools allow spawning specialized sub-agents as subprocesses
-        for focused tasks like exploration and planning.
+        """Register sub-agent and background task management tools.
+
+        These tools allow:
+        - Spawning specialized sub-agents as subprocesses
+        - Running shell commands in the background
+        - Getting output from background tasks
+        - Killing background tasks
+        - Listing all background tasks
         """
         from .tools.task import TaskTool
-        from .tools.task_output import TaskOutputTool
+        from .tools.task_output import TaskOutputTool, KillTaskTool, ListTasksTool
 
         self.register_tool(TaskTool(repo_root=self._repo_root, connection=self.connection))
         self.register_tool(TaskOutputTool(repo_root=self._repo_root, connection=self.connection))
+        self.register_tool(KillTaskTool(repo_root=self._repo_root, connection=self.connection))
+        self.register_tool(ListTasksTool(repo_root=self._repo_root, connection=self.connection))
 
     def _register_mode_tools(self) -> None:
         """Register mode switching tools.
@@ -347,10 +375,58 @@ class AgentToolkit:
             if tools:
                 log.info(f"Registered {len(tools)} dynamic MCP tools from config")
 
+            # Register LSP tools if USE_LSP is enabled (default: true)
+            self._register_lsp_tools()
+
         except Exception as e:
             log.warning(f"Failed to initialize MCP manager: {e}")
             self._mcp_manager = None
 
+    def _register_lsp_tools(self) -> None:
+        """Register LSP-based code navigation tools.
+
+        These tools use cclsp MCP server to provide IDE-level code intelligence.
+        Enabled by default with USE_LSP=true. Set USE_LSP=false to disable.
+        """
+        from .tools.lsp import (
+            is_lsp_enabled,
+            LSPFindDefinitionTool,
+            LSPFindReferencesTool,
+            LSPRenameSymbolTool,
+            LSPGetDiagnosticsTool,
+        )
+
+        if not is_lsp_enabled():
+            log.info("LSP tools disabled (USE_LSP=false)")
+            return
+
+        if self._mcp_manager is None:
+            log.warning("Cannot register LSP tools: MCP manager not initialized")
+            return
+
+        # Check if cclsp is available
+        config = self._mcp_manager.load_config()
+        cclsp_config = config.get_server("cclsp")
+        if not cclsp_config or not cclsp_config.enabled:
+            log.info("LSP tools not registered: cclsp MCP server not enabled")
+            return
+
+        # Register LSP tools with better descriptions for the agent
+        lsp_tools = [
+            LSPFindDefinitionTool(self._mcp_manager, self.connection),
+            LSPFindReferencesTool(self._mcp_manager, self.connection),
+            LSPRenameSymbolTool(self._mcp_manager, self.connection),
+            LSPGetDiagnosticsTool(self._mcp_manager, self.connection),
+        ]
+
+        for tool in lsp_tools:
+            # LSP tools take priority - overwrite if exists
+            if tool.name in self._tools:
+                log.debug(f"LSP tool '{tool.name}' overwriting existing tool")
+            self.register_tool(tool)
+
+        log.info(f"Registered {len(lsp_tools)} LSP tools (USE_LSP=true)")
+
     def get_mcp_manager(self):
         """Get the MCP manager instance.