applied-cli 0.1.0__py3-none-any.whl

applied_cli/mcp_server.py

@@ -0,0 +1,738 @@
+"""
+MCP Server for Applied Labs CLI.
+
+This module exposes applied-cli commands as MCP tools for use with Claude.
+
+Usage:
+    # Run directly
+    python -m applied_cli.mcp_server
+
+    # Or via the entry point
+    applied-cli-mcp
+"""
+
+import json
+import subprocess
+import sys
+from typing import Any
+
+from mcp.server.fastmcp import FastMCP
+
+# Initialize FastMCP server
+mcp = FastMCP(
+    "applied-cli",
+    instructions="""
+You have access to the Applied Labs CLI for managing AI support agents.
+
+## Quick Reference
+
+### Setup & Auth
+- `auth_status` - Check authentication and current shop
+- `auth_shops` / `auth_switch_shop` - List and switch shops
+- `shop_setup` - Full shop bootstrap from YAML spec
+
+### Agents
+- `agent_list` / `agent_show` - View agents
+- `agent_create` / `agent_update` - Manage agents
+- `chat_send` - Test agent responses
+
+### Knowledge Base
+- `knowledge_list` - List Q&A entries and rules
+- `knowledge_upsert` - Create/update entries
+
+### Testing Workflow
+1. `test_benchmarks_list` - List benchmarks
+2. `test_fix_context` - Get failing scenarios with feedback
+3. `knowledge_upsert` - Fix knowledge base
+4. `test_fix_batch` - Re-test scenarios in bulk
+5. `test_fix_status` - Track progress
+6. `test_coverage_summary` - Coverage by intent
+
+### Data
+- `conversations_list` / `conversations_show` - View conversations
+- `conversations_import` - Import historical CSV
+- `simulate_run` - Generate test conversations
+- `insights_run` / `insights_show` - Analytics reports
+
+All tools return JSON.
+""",
+)
+
+
+def _run_cli(args: list[str], timeout: int = 120) -> dict[str, Any]:
+    """Run an applied-cli command and return structured output."""
+    cmd = ["python", "-m", "applied_cli.main"] + args
+
+    try:
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+
+        output = result.stdout.strip()
+        error = result.stderr.strip()
+
+        # Try to parse as JSON
+        try:
+            parsed = json.loads(output)
+            return {
+                "success": result.returncode == 0,
+                "data": parsed,
+                "error": error if result.returncode != 0 else None,
+            }
+        except json.JSONDecodeError:
+            return {
+                "success": result.returncode == 0,
+                "output": output,
+                "error": error if result.returncode != 0 else None,
+            }
+
+    except subprocess.TimeoutExpired:
+        return {
+            "success": False,
+            "error": f"Command timed out after {timeout} seconds",
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+        }
+
+
+# =============================================================================
+# AUTHENTICATION
+# =============================================================================
+
+
+@mcp.tool()
+async def auth_status() -> str:
+    """Check authentication status and current shop."""
+    result = _run_cli(["auth", "status", "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def auth_shops() -> str:
+    """List all shops the user has access to."""
+    result = _run_cli(["auth", "shops", "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def auth_switch_shop(shop_identifier: str) -> str:
+    """
+    Switch to a different shop.
+
+    Args:
+        shop_identifier: Shop name or UUID to switch to
+    """
+    result = _run_cli(["auth", "use-shop", shop_identifier])
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# AGENTS
+# =============================================================================
+
+
+@mcp.tool()
+async def agent_list() -> str:
+    """List all agents in the current shop."""
+    result = _run_cli(["agent", "list", "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def agent_show(agent_id: str) -> str:
+    """
+    Get detailed information about a specific agent.
+
+    Args:
+        agent_id: UUID of the agent
+    """
+    result = _run_cli(["agent", "show", "--agent-id", agent_id, "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def agent_create(
+    name: str,
+    modality: str,
+    description: str = "",
+    agent_type: str = "customer_support",
+) -> str:
+    """
+    Create a new agent.
+
+    Args:
+        name: Name for the agent
+        modality: 'chat', 'email', or 'sms'
+        description: Optional description
+        agent_type: Agent type (default: customer_support)
+    """
+    args = ["agent", "create", "--name", name, "--modality", modality, "--type", agent_type, "--json"]
+    if description:
+        args.extend(["--description", description])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def agent_update(
+    agent_id: str,
+    name: str = None,
+    description: str = None,
+    guardrail: str = None,
+) -> str:
+    """
+    Update an existing agent.
+
+    Args:
+        agent_id: UUID of the agent
+        name: New name
+        description: New description
+        guardrail: New guardrail text
+    """
+    args = ["agent", "update", "--agent-id", agent_id, "--yes", "--json"]
+    if name:
+        args.extend(["--name", name])
+    if description:
+        args.extend(["--description", description])
+    if guardrail:
+        args.extend(["--guardrail", guardrail])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# CHAT
+# =============================================================================
+
+
+@mcp.tool()
+async def chat_send(agent_id: str, message: str, channel: str = "chat") -> str:
+    """
+    Send a message to an agent and get the response.
+
+    Args:
+        agent_id: UUID of the agent to chat with
+        message: The message to send
+        channel: Channel type - 'chat', 'email', or 'sms' (default: chat)
+    """
+    result = _run_cli(
+        ["chat", "--agent-id", agent_id, "--channel", channel, "--message", message, "--json"],
+        timeout=90,
+    )
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# CONVERSATIONS
+# =============================================================================
+
+
+@mcp.tool()
+async def conversations_list(limit: int = 20, agent_id: str = None) -> str:
+    """
+    List recent conversations.
+
+    Args:
+        limit: Maximum number of conversations to return (default: 20)
+        agent_id: Optional agent UUID to filter by
+    """
+    args = ["conversations", "list", "--limit", str(limit), "--json"]
+    if agent_id:
+        args.extend(["--agent-id", agent_id])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def conversations_show(conversation_id: str) -> str:
+    """
+    Get full details of a conversation including all messages.
+
+    Args:
+        conversation_id: UUID of the conversation
+    """
+    result = _run_cli(["conversations", "show", "--conversation-id", conversation_id, "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def conversations_import(
+    agent_id: str,
+    file_path: str = None,
+    url: str = None,
+) -> str:
+    """
+    Import historical conversations from CSV.
+
+    Provide either file_path or url. CSV should have columns:
+    ID, DATE_CREATED, SENDER, BODY, TOPIC, INTENT.
+
+    Args:
+        agent_id: UUID of the agent to import for
+        file_path: Local path to CSV file
+        url: URL to CSV file
+    """
+    args = ["conversations", "import", "--agent-id", agent_id, "--yes", "--json"]
+    if file_path:
+        args.extend(["--file-path", file_path])
+    elif url:
+        args.extend(["--url", url])
+    result = _run_cli(args, timeout=300)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# KNOWLEDGE BASE
+# =============================================================================
+
+
+@mcp.tool()
+async def knowledge_list(agent_id: str, entry_type: str = None) -> str:
+    """
+    List knowledge base entries for an agent.
+
+    Entry types:
+    - 'escalate': Triggers escalation to human
+    - 'qa': Question-answer pairs
+    - 'exact': Returns answer verbatim
+    - 'context': Background knowledge
+
+    Args:
+        agent_id: UUID of the agent
+        entry_type: Optional filter - 'escalate', 'qa', 'exact', or 'context'
+    """
+    args = ["knowledge", "list", "--agent-id", agent_id, "--json"]
+    if entry_type:
+        args.extend(["--type", entry_type])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def knowledge_upsert(
+    agent_id: str,
+    entry_type: str,
+    question: str,
+    answer: str = "",
+) -> str:
+    """
+    Create or update a knowledge base entry.
+
+    Args:
+        agent_id: UUID of the agent
+        entry_type: Type - 'escalate', 'qa', 'exact', 'context'
+        question: The question/trigger to match
+        answer: The answer content
+    """
+    args = [
+        "knowledge", "upsert",
+        "--agent-id", agent_id,
+        "--type", entry_type,
+        "--question", question,
+        "--yes", "--json",
+    ]
+    if answer:
+        args.extend(["--answer", answer])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# TAXONOMY
+# =============================================================================
+
+
+@mcp.tool()
+async def taxonomy_list() -> str:
+    """List all topics and intents in the shop taxonomy."""
+    result = _run_cli(["taxonomy", "list", "--json"])
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# SHOP SETUP
+# =============================================================================
+
+
+@mcp.tool()
+async def shop_setup(spec_path: str, shop_id: str = None) -> str:
+    """
+    Full shop setup from a YAML spec file.
+
+    Creates shop, configures agents, uploads conversations,
+    runs insights, simulation, and syncs knowledge base.
+
+    Args:
+        spec_path: Path to YAML spec file
+        shop_id: Optional existing shop UUID to configure instead of creating new
+    """
+    args = ["shop", "setup", "--spec", spec_path, "--yes", "--json"]
+    if shop_id:
+        args.extend(["--shop-id", shop_id])
+    result = _run_cli(args, timeout=600)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# SIMULATE
+# =============================================================================
+
+
+@mcp.tool()
+async def simulate_run(
+    agent_id: str,
+    count: int = 10,
+    conversation_type: str = "web_chat",
+    is_test: bool = False,
+) -> str:
+    """
+    Generate simulated test conversations for an agent.
+
+    Args:
+        agent_id: UUID of the agent
+        count: Number of conversations to generate
+        conversation_type: 'web_chat', 'email', or 'sms'
+        is_test: Mark as test conversations
+    """
+    args = [
+        "simulate", "run",
+        "--agent-id", agent_id,
+        "--count", str(count),
+        "--conversation-type", conversation_type,
+        "--continue-on-error",
+        "--json",
+    ]
+    if is_test:
+        args.append("--is-test")
+    else:
+        args.append("--no-is-test")
+    result = _run_cli(args, timeout=600)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# TEST: BENCHMARKS
+# =============================================================================
+
+
+@mcp.tool()
+async def test_benchmarks_list() -> str:
+    """List all test benchmarks in the current shop."""
+    result = _run_cli(["test", "benchmarks", "list", "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_benchmarks_create(name: str, agent_id: str) -> str:
+    """
+    Create a new benchmark.
+
+    Args:
+        name: Name for the benchmark
+        agent_id: UUID of the agent this benchmark is for
+    """
+    result = _run_cli(["test", "benchmarks", "create", "--name", name, "--agent-id", agent_id, "--json"])
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# TEST: SCENARIOS
+# =============================================================================
+
+
+@mcp.tool()
+async def test_scenarios_list(
+    benchmark_id: str,
+    pass_status: str = None,
+    limit: int = 100,
+) -> str:
+    """
+    List test scenarios in a benchmark.
+
+    Args:
+        benchmark_id: UUID of the benchmark
+        pass_status: Optional filter - 'pass', 'fail', or 'unrated'
+        limit: Maximum scenarios to return (default: 100)
+    """
+    args = ["test", "scenarios", "list", "--benchmark-id", benchmark_id, "--limit", str(limit), "--json"]
+    if pass_status:
+        args.extend(["--pass-status", pass_status])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_scenarios_show(scenario_id: str) -> str:
+    """
+    Get full details of a test scenario.
+
+    Args:
+        scenario_id: UUID of the scenario
+    """
+    result = _run_cli(["test", "scenarios", "show", "--scenario-id", scenario_id, "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_scenarios_update(
+    scenario_id: str,
+    pass_status: str = None,
+    feedback: str = None,
+) -> str:
+    """
+    Update a scenario's pass status or feedback.
+
+    Args:
+        scenario_id: UUID of the scenario
+        pass_status: New status - 'pass' or 'fail'
+        feedback: Optional feedback notes
+    """
+    args = ["test", "scenarios", "update", "--scenario-id", scenario_id]
+    if pass_status:
+        args.extend(["--pass-status", pass_status])
+    if feedback:
+        args.extend(["--feedback", feedback])
+    result = _run_cli(args)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_scenarios_rate(
+    conversation_id: str,
+    agent_id: str = None,
+    auto: bool = True,
+) -> str:
+    """
+    Rate a conversation and create a test scenario from it.
+
+    Args:
+        conversation_id: UUID of the conversation to rate
+        agent_id: Optional agent UUID override
+        auto: Auto-compute rating (default: True)
+    """
+    args = ["test", "scenarios", "rate", "--conversation-id", conversation_id, "--yes", "--json"]
+    if agent_id:
+        args.extend(["--agent-id", agent_id])
+    if not auto:
+        args.append("--manual")
+    result = _run_cli(args, timeout=60)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# TEST: COVERAGE
+# =============================================================================
+
+
+@mcp.tool()
+async def test_coverage_summary(agent_id: str, benchmark_id: str = None) -> str:
+    """
+    Summarize test coverage by topic/intent.
+
+    Args:
+        agent_id: UUID of the agent
+        benchmark_id: Optional benchmark UUID to filter by
+    """
+    args = ["test", "coverage", "summary", "--agent-id", agent_id, "--json"]
+    if benchmark_id:
+        args.extend(["--benchmark-id", benchmark_id])
+    result = _run_cli(args, timeout=120)
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# TEST: FIX
+# =============================================================================
+
+
+@mcp.tool()
+async def test_fix_context(benchmark_id: str, include_passing: bool = False) -> str:
+    """
+    Get all context needed to fix failing scenarios.
+
+    Returns failing scenarios with feedback, knowledge base entries,
+    and agent guardrails.
+
+    Args:
+        benchmark_id: UUID of the benchmark to analyze
+        include_passing: Also include passing scenarios for reference
+    """
+    args = ["test", "fix", "context", "--benchmark-id", benchmark_id, "--json"]
+    if include_passing:
+        args.append("--include-passing")
+    result = _run_cli(args, timeout=180)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_fix_test(
+    scenario_id: str,
+    benchmark_id: str,
+    auto_pass: bool = False,
+    expect_escalation: bool = None,
+) -> str:
+    """
+    Test a fix by replaying a scenario's input message.
+
+    Args:
+        scenario_id: Original scenario UUID to replay
+        benchmark_id: Target benchmark UUID for the new scenario
+        auto_pass: Automatically mark the new scenario as pass
+        expect_escalation: If True, fail if agent doesn't escalate
+    """
+    args = [
+        "test", "fix", "test",
+        "--scenario-id", scenario_id,
+        "--benchmark-id", benchmark_id,
+        "--retry", "2", "--json",
+    ]
+    if auto_pass:
+        args.append("--auto-pass")
+    if expect_escalation is True:
+        args.append("--expect-escalation")
+    elif expect_escalation is False:
+        args.append("--expect-response")
+    result = _run_cli(args, timeout=120)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_fix_batch(
+    source_benchmark_id: str,
+    target_benchmark_id: str,
+    pass_status_filter: str = "fail",
+    auto_pass: bool = False,
+    limit: int = 0,
+    parallel: int = 1,
+) -> str:
+    """
+    Batch test scenarios from one benchmark into another.
+
+    Args:
+        source_benchmark_id: Source benchmark with scenarios to test
+        target_benchmark_id: Target benchmark for new test scenarios
+        pass_status_filter: Filter - 'fail', 'pass', 'unrated', or 'all'
+        auto_pass: Automatically mark successful tests as pass
+        limit: Max scenarios to test, 0 = all
+        parallel: Number of parallel tests 1-10
+    """
+    args = [
+        "test", "fix", "batch",
+        "--source", source_benchmark_id,
+        "--target", target_benchmark_id,
+        "--pass-status", pass_status_filter,
+        "--parallel", str(min(10, max(1, parallel))),
+        "--retry", "2", "--json",
+    ]
+    if auto_pass:
+        args.append("--auto-pass")
+    if limit > 0:
+        args.extend(["--limit", str(limit)])
+    result = _run_cli(args, timeout=600)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_fix_status(source_benchmark_id: str, target_benchmark_id: str) -> str:
+    """
+    Track fix progress between benchmarks.
+
+    Args:
+        source_benchmark_id: Source benchmark with original scenarios
+        target_benchmark_id: Target benchmark with test results
+    """
+    result = _run_cli([
+        "test", "fix", "status",
+        "--source", source_benchmark_id,
+        "--target", target_benchmark_id,
+        "--json",
+    ])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def test_fix_summary(benchmark_id: str) -> str:
+    """
+    Get a quick summary of benchmark pass/fail status.
+
+    Args:
+        benchmark_id: UUID of the benchmark
+    """
+    result = _run_cli(["test", "fix", "summary", "--benchmark-id", benchmark_id, "--json"])
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# INSIGHTS
+# =============================================================================
+
+
+@mcp.tool()
+async def insights_run(
+    query: str,
+    date_range: str = "relative:-30d,",
+    agent_ids: str = None,
+    wait: bool = True,
+) -> str:
+    """
+    Generate an analytics insights report.
+
+    Args:
+        query: Natural language query like "What are the top customer issues?"
+        date_range: Date range - 'relative:-30d,' for last 30 days
+        agent_ids: Comma-separated agent UUIDs to filter by
+        wait: Wait for report completion
+    """
+    args = ["insights", "run", "--query", query, "--date-range", date_range, "--json"]
+    if agent_ids:
+        args.extend(["--agent-ids", agent_ids])
+    if wait:
+        args.append("--wait")
+    result = _run_cli(args, timeout=300)
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def insights_list(limit: int = 10) -> str:
+    """
+    List recent insights reports.
+
+    Args:
+        limit: Maximum reports to return
+    """
+    result = _run_cli(["insights", "list", "--limit", str(limit), "--json"])
+    return json.dumps(result, indent=2)
+
+
+@mcp.tool()
+async def insights_show(report_id: str) -> str:
+    """
+    Get the full content of an insights report.
+
+    Args:
+        report_id: UUID of the report
+    """
+    result = _run_cli(["insights", "show", "--report-id", report_id, "--json"])
+    return json.dumps(result, indent=2)
+
+
+# =============================================================================
+# ENTRY POINT
+# =============================================================================
+
+
+def main():
+    """Run the MCP server."""
+    import logging
+    logging.basicConfig(level=logging.WARNING, stream=sys.stderr)
+    mcp.run(transport="stdio")
+
+
+if __name__ == "__main__":
+    main()