npcsh 1.1.16__py3-none-any.whl → 1.1.17__py3-none-any.whl

npcsh/_state.py

@@ -351,8 +351,7 @@ def initialize_base_npcs_if_needed(db_path: str) -> None:
         None
     """
 
-    if is_npcsh_initialized():
-        return
+    already_initialized = is_npcsh_initialized()
 
     conn = sqlite3.connect(db_path)
     cursor = conn.cursor()
@@ -508,8 +507,10 @@ def initialize_base_npcs_if_needed(db_path: str) -> None:
                         print(f"Copied template {file} to {destination_template_path}")
     conn.commit()
     conn.close()
-    set_npcsh_initialized()
-    add_npcshrc_to_shell_config()
+
+    if not already_initialized:
+        set_npcsh_initialized()
+        add_npcshrc_to_shell_config()
 
 
 def get_shell_config_file() -> str:
@@ -2540,8 +2541,19 @@ def collect_llm_tools(state: ShellState) -> Tuple[List[Dict[str, Any]], Dict[str
         if not jinja_env_for_jinx and state.team and isinstance(state.team, Team):
             jinja_env_for_jinx = getattr(state.team, "jinja_env", None)
 
+        jinx_globals = {
+            "state": state,
+            "CommandHistory": CommandHistory,
+            "load_kg_from_db": load_kg_from_db,
+            "execute_rag_command": execute_rag_command,
+            "execute_brainblast_command": execute_brainblast_command,
+            "load_file_contents": load_file_contents,
+            "search_web": search_web,
+            "get_relevant_memories": get_relevant_memories,
+        }
+
         for name, jinx_obj in aggregated_jinxs.items():
-            def _make_runner(jinx=jinx_obj, jinja_env=jinja_env_for_jinx, tool_name=name):
+            def _make_runner(jinx=jinx_obj, jinja_env=jinja_env_for_jinx, tool_name=name, extras=jinx_globals):
                 def runner(**kwargs):
                     input_values = kwargs if isinstance(kwargs, dict) else {}
                     try:
@@ -2549,7 +2561,7 @@ def collect_llm_tools(state: ShellState) -> Tuple[List[Dict[str, Any]], Dict[str
                             input_values=input_values,
                             npc=npc_obj,
                             messages=state.messages,
-                            extra_globals={"state": state},
+                            extra_globals=extras,
                             jinja_env=jinja_env
                         )
                         return ctx.get("output", ctx)
@@ -2945,14 +2957,17 @@ def process_pipeline_command(
                                 tool_name = msg.get("name", "tool")
                                 tool_content = msg.get("content", "")
                                 if tool_content and tool_content.strip():
+                                    # Decode escaped newlines if present
+                                    if isinstance(tool_content, str):
+                                        tool_content = tool_content.replace('\\n', '\n').replace('\\t', '\t')
                                     print(colored(f"\n⚡ {tool_name}:", "cyan"))
                                     lines = tool_content.split('\n')
                                     if len(lines) > 50:
-                                        print('\n'.join(lines[:25]))
+                                        render_markdown('\n'.join(lines[:25]))
                                         print(colored(f"\n... ({len(lines) - 50} lines hidden) ...\n", "white", attrs=["dark"]))
-                                        print('\n'.join(lines[-25:]))
+                                        render_markdown('\n'.join(lines[-25:]))
                                     else:
-                                        print(tool_content)
+                                        render_markdown(tool_content)
 
                         # Check if LLM made tool calls - if not, it's done
                         tool_calls_made = isinstance(llm_result, dict) and llm_result.get("tool_calls")

npcsh/benchmark/__init__.py

@@ -0,0 +1,22 @@
+"""
+npcsh benchmark integration for Terminal-Bench.
+
+This module provides integration with Terminal-Bench (tbench.ai) for benchmarking
+npcsh against standardized terminal/CLI agent evaluation tasks.
+
+Usage:
+    # Install terminal-bench
+    pip install terminal-bench harbor
+
+    # Run benchmarks with npcsh
+    harbor run -d terminal-bench@2.0 --agent-import-path npcsh.benchmark:NpcshAgent -m anthropic/claude-sonnet-4-20250514
+
+    # Or use the convenience function
+    from npcsh.benchmark import run_benchmark
+    run_benchmark(model="claude-sonnet-4-20250514", provider="anthropic")
+"""
+
+from .npcsh_agent import NpcshAgent
+from .runner import run_benchmark, BenchmarkRunner
+
+__all__ = ["NpcshAgent", "run_benchmark", "BenchmarkRunner"]

npcsh/benchmark/npcsh_agent.py

@@ -0,0 +1,262 @@
+"""
+npcsh Harbor Agent Adapter for Terminal-Bench.
+
+This module implements the BaseInstalledAgent interface for running npcsh
+as an agent in Terminal-Bench evaluations.
+"""
+
+import json
+import os
+import shlex
+from pathlib import Path
+from typing import Optional
+
+from harbor.agents.installed.base import BaseInstalledAgent, ExecInput
+from harbor.models.agent.context import AgentContext
+
+
+class NpcshAgent(BaseInstalledAgent):
+    """
+    Harbor agent adapter for npcsh.
+
+    This allows npcsh to be evaluated on Terminal-Bench tasks by:
+    1. Installing npcsh in the benchmark container
+    2. Running npcsh with the task instruction
+    3. Parsing output for token usage and results
+
+    Usage:
+        harbor run -d terminal-bench@2.0 \\
+            --agent-import-path npcsh.benchmark:NpcshAgent \\
+            -m anthropic/claude-sonnet-4-20250514 -n 4
+    """
+
+    SUPPORTS_ATIF = True  # Agent Trajectory Interchange Format
+
+    def __init__(self, logs_dir: Path = None, model_name: str = None, logger=None, **kwargs):
+        super().__init__(logs_dir=logs_dir, model_name=model_name, logger=logger, **kwargs)
+
+    @staticmethod
+    def name() -> str:
+        return "npcsh"
+
+    @property
+    def _install_agent_template_path(self) -> Path:
+        """Path to the jinja template script for installing npcsh in the container."""
+        return Path(__file__).parent / "templates" / "install-npcsh.sh.j2"
+
+    def create_run_agent_commands(self, instruction: str) -> list:
+        """
+        Create the commands to run npcsh in the container.
+
+        Args:
+            instruction: The task instruction from Terminal-Bench
+
+        Returns:
+            List of ExecInput commands to execute
+        """
+        escaped_instruction = shlex.quote(instruction)
+        model_name = self.model_name
+
+        if model_name and "/" in model_name:
+            provider, model = model_name.split("/", 1)
+        elif model_name:
+            provider = os.environ.get("NPCSH_CHAT_PROVIDER", "")
+            model = model_name
+        else:
+            provider = os.environ.get("NPCSH_CHAT_PROVIDER", "")
+            model = os.environ.get("NPCSH_CHAT_MODEL", "")
+
+        # Map provider names to npcsh provider format
+        provider_map = {
+            "anthropic": "anthropic",
+            "openai": "openai",
+            "google": "gemini",
+            "gemini": "gemini",
+            "deepseek": "deepseek",
+            "ollama": "ollama",
+            "groq": "groq",
+            "openrouter": "openrouter",
+        }
+        npcsh_provider = provider_map.get(provider.lower(), provider)
+
+        # Build environment variables for API keys
+        env_vars = []
+        api_key_map = {
+            "anthropic": "ANTHROPIC_API_KEY",
+            "openai": "OPENAI_API_KEY",
+            "gemini": "GOOGLE_API_KEY",
+            "google": "GOOGLE_API_KEY",
+            "deepseek": "DEEPSEEK_API_KEY",
+            "groq": "GROQ_API_KEY",
+            "openrouter": "OPENROUTER_API_KEY",
+        }
+
+        for prov, env_key in api_key_map.items():
+            if env_key in os.environ:
+                env_vars.append(f'{env_key}="{os.environ[env_key]}"')
+
+        env_prefix = " ".join(env_vars) + " " if env_vars else ""
+
+        # Output directory for logs
+        output_dir = str(self.logs_dir / "npcsh_output")
+        output_file = str(self.logs_dir / "npcsh_output" / "output.jsonl")
+
+        commands = []
+
+        # Create output directory
+        commands.append(ExecInput(
+            cmd=f"mkdir -p {shlex.quote(output_dir)}",
+            timeout=30
+        ))
+
+        # Run npcsh with the instruction
+        # Using the npc CLI which supports single-command execution
+        npcsh_cmd = (
+            f'{env_prefix}'
+            f'NPCSH_CHAT_MODEL="{model}" '
+            f'NPCSH_CHAT_PROVIDER="{npcsh_provider}" '
+            f'NPCSH_STREAM_OUTPUT=0 '
+            f'npc {escaped_instruction} '
+            f'2>&1 | tee {shlex.quote(output_file)}'
+        )
+
+        commands.append(ExecInput(
+            cmd=npcsh_cmd,
+            timeout=600,  # 10 minute timeout for complex tasks
+        ))
+
+        return commands
+
+    def populate_context_post_run(self, context: AgentContext) -> None:
+        """
+        Populate the context with results of the agent execution.
+
+        Parses the output file to extract token usage metrics.
+
+        Args:
+            context: The AgentContext to populate with metrics
+        """
+        output_file = self.logs_dir / "npcsh_output" / "output.jsonl"
+
+        total_input_tokens = 0
+        total_output_tokens = 0
+        total_cost_usd = 0.0
+
+        if output_file.exists():
+            try:
+                with open(output_file, 'r') as f:
+                    content = f.read()
+
+                # Try to parse as JSONL first
+                for line in content.strip().split('\n'):
+                    if not line.strip():
+                        continue
+                    try:
+                        event = json.loads(line)
+                        # Extract token usage from events if present
+                        if isinstance(event, dict):
+                            usage = event.get('usage', {})
+                            total_input_tokens += usage.get('input_tokens', 0)
+                            total_output_tokens += usage.get('output_tokens', 0)
+                            total_cost_usd += usage.get('cost_usd', 0.0)
+                    except json.JSONDecodeError:
+                        # Not JSON, just regular output
+                        pass
+
+            except Exception as e:
+                self.logger.warning(f"Failed to parse npcsh output: {e}")
+
+        # Set context metrics
+        if hasattr(context, 'input_tokens'):
+            context.input_tokens = total_input_tokens
+        if hasattr(context, 'output_tokens'):
+            context.output_tokens = total_output_tokens
+        if hasattr(context, 'cost_usd'):
+            context.cost_usd = total_cost_usd
+
+
+class NpcshAgentWithNpc(NpcshAgent):
+    """
+    Variant that uses a specific NPC for task execution.
+
+    This allows benchmarking specific NPCs like sibiji (orchestrator),
+    corca (coding), or custom NPCs.
+
+    Usage:
+        harbor run -d terminal-bench@2.0 \\
+            --agent-import-path "npcsh.benchmark:NpcshAgentWithNpc" \\
+            -m anthropic/claude-sonnet-4-20250514 -n 4
+    """
+
+    def __init__(self, *args, npc_name: str = "sibiji", **kwargs):
+        super().__init__(*args, **kwargs)
+        self.npc_name = npc_name
+
+    @staticmethod
+    def name() -> str:
+        return "npcsh-npc"
+
+    def create_run_agent_commands(self, instruction: str) -> list:
+        """Create commands using a specific NPC."""
+        escaped_instruction = shlex.quote(instruction)
+        model_name = self.model_name
+
+        if model_name and "/" in model_name:
+            provider, model = model_name.split("/", 1)
+        elif model_name:
+            provider = os.environ.get("NPCSH_CHAT_PROVIDER", "")
+            model = model_name
+        else:
+            provider = os.environ.get("NPCSH_CHAT_PROVIDER", "")
+            model = os.environ.get("NPCSH_CHAT_MODEL", "")
+
+        provider_map = {
+            "anthropic": "anthropic",
+            "openai": "openai",
+            "google": "gemini",
+            "gemini": "gemini",
+            "deepseek": "deepseek",
+            "ollama": "ollama",
+        }
+        npcsh_provider = provider_map.get(provider.lower(), provider)
+
+        env_vars = []
+        api_key_map = {
+            "anthropic": "ANTHROPIC_API_KEY",
+            "openai": "OPENAI_API_KEY",
+            "gemini": "GOOGLE_API_KEY",
+            "deepseek": "DEEPSEEK_API_KEY",
+        }
+
+        for prov, env_key in api_key_map.items():
+            if env_key in os.environ:
+                env_vars.append(f'{env_key}="{os.environ[env_key]}"')
+
+        env_prefix = " ".join(env_vars) + " " if env_vars else ""
+
+        output_dir = str(self.logs_dir / "npcsh_output")
+        output_file = str(self.logs_dir / "npcsh_output" / "output.jsonl")
+
+        commands = []
+
+        commands.append(ExecInput(
+            cmd=f"mkdir -p {shlex.quote(output_dir)}",
+            timeout=30
+        ))
+
+        # Use specific NPC with --npc flag
+        npcsh_cmd = (
+            f'{env_prefix}'
+            f'NPCSH_CHAT_MODEL="{model}" '
+            f'NPCSH_CHAT_PROVIDER="{npcsh_provider}" '
+            f'NPCSH_STREAM_OUTPUT=0 '
+            f'npc --npc {self.npc_name} {escaped_instruction} '
+            f'2>&1 | tee {shlex.quote(output_file)}'
+        )
+
+        commands.append(ExecInput(
+            cmd=npcsh_cmd,
+            timeout=600,
+        ))
+
+        return commands