icdev 0.0.3__py3-none-any.whl

tools/mcp/tool_registry.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Lazy MCP Tool Registry — loads tool definitions on-demand based on session purpose.
+
+Maps declared session purposes (from tools/agent/session_purpose.py) to relevant
+MCP server subsets, reducing context window overhead by only loading tools
+the current session actually needs.
+
+ADRs: D-ARCH-9 (lazy tool loading by session purpose)
+
+CLI:
+    python tools/mcp/tool_registry.py --purpose compliance --json
+    python tools/mcp/tool_registry.py --server builder --json
+    python tools/mcp/tool_registry.py --all --json
+    python tools/mcp/tool_registry.py --savings compliance --json
+"""
+
+import argparse
+import json
+import sys
+from typing import Dict, List, Optional
+
+
+# ---------------------------------------------------------------------------
+# MCP Server Definitions (12 servers, tool names only)
+# ---------------------------------------------------------------------------
+
+MCP_SERVERS: Dict[str, List[str]] = {
+    "architect": [
+        "design_system", "decompose", "interface_contract",
+    ],
+    "builder": [
+        "scaffold", "generate_code", "write_tests", "run_tests", "lint", "format",
+    ],
+    "compliance": [
+        "ssp_generate", "poam_generate", "stig_check", "sbom_generate",
+        "cui_mark", "control_map", "nist_lookup",
+    ],
+    "core": [
+        "project_create", "project_list", "project_status",
+        "task_dispatch", "agent_status",
+    ],
+    "devsecops": [
+        "devsecops_profile_create", "zta_maturity_score",
+        "pipeline_security_generate", "policy_generate",
+        "service_mesh_generate",
+    ],
+    "knowledge": [
+        "search_knowledge", "add_pattern", "get_recommendations", "self_heal",
+    ],
+    "monitor": [
+        "log_analyze", "health_check", "metrics_query", "alert_manage",
+    ],
+    "requirements": [
+        "create_intake_session", "process_intake_turn", "detect_gaps",
+        "score_readiness", "decompose_requirements",
+    ],
+    "security": [
+        "sast_scan", "dep_audit", "secret_detect", "container_scan",
+    ],
+    "simulation": [
+        "create_scenario", "run_simulation", "run_monte_carlo",
+        "generate_coas", "compare_coas",
+    ],
+    "supply-chain": [
+        "add_vendor", "build_dependency_graph", "assess_scrm",
+        "triage_cve", "manage_isa",
+    ],
+    "connector-forge": [
+        "forge_from_spec", "forge_from_yaml", "forge_status",
+        "forge_validate", "forge_promote", "forge_publish",
+        "forge_import", "forge_list",
+    ],
+}
+
+# ---------------------------------------------------------------------------
+# Purpose-to-server mapping
+# ---------------------------------------------------------------------------
+
+PURPOSE_MAP: Dict[str, List[str]] = {
+    "compliance":    ["compliance", "security", "devsecops"],
+    "security":      ["security", "compliance", "devsecops"],
+    "build":         ["builder", "core", "knowledge"],
+    "requirements":  ["requirements", "simulation", "supply-chain"],
+    "deploy":        ["devsecops", "core", "builder"],
+    "monitor":       ["monitor", "core", "knowledge"],
+    "design":        ["architect", "core", "requirements"],
+    "mbse":          ["requirements", "core", "compliance"],
+    "govcon":        ["core", "compliance", "knowledge"],
+    "test":          ["builder", "security", "compliance"],
+    "general":       ["core", "builder", "compliance"],
+}
+
+# Keywords used by resolve_purpose() for fuzzy matching
+_PURPOSE_KEYWORDS: Dict[str, List[str]] = {
+    "compliance":   ["compliance", "ato", "fedramp", "nist", "cmmc", "stig", "poam", "ssp"],
+    "security":     ["security", "sast", "vulnerability", "secret", "scan", "threat", "pentest"],
+    "build":        ["build", "code", "scaffold", "generate", "tdd", "implement"],
+    "requirements": ["requirement", "intake", "gap", "readiness", "decompose", "ricoas"],
+    "deploy":       ["deploy", "pipeline", "cicd", "release", "devops", "zta"],
+    "monitor":      ["monitor", "log", "metric", "alert", "health", "observe"],
+    "design":       ["design", "architect", "interface", "decompose", "system"],
+    "mbse":         ["mbse", "sysml", "reqif", "model", "thread", "traceability"],
+    "govcon":       ["govcon", "proposal", "rfp", "contract", "cpars", "sam.gov"],
+    "test":         ["test", "qa", "verify", "validate", "coverage", "behave"],
+}
+
+# ---------------------------------------------------------------------------
+# Total tool count (cached)
+# ---------------------------------------------------------------------------
+
+_TOTAL_TOOLS: int = sum(len(tools) for tools in MCP_SERVERS.values())
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def get_tools_for_purpose(purpose: str) -> Dict:
+    """Return MCP servers and tools relevant to a session purpose.
+
+    Args:
+        purpose: A purpose category key (e.g. 'compliance', 'build').
+                 Falls back to 'general' if not recognised.
+
+    Returns:
+        Dict with servers (list of {name, tools}), total_tools, purpose.
+    """
+    key = purpose.lower().strip()
+    server_names = PURPOSE_MAP.get(key, PURPOSE_MAP["general"])
+
+    servers = []
+    total = 0
+    for name in server_names:
+        tools = MCP_SERVERS.get(name, [])
+        servers.append({"name": name, "tools": tools})
+        total += len(tools)
+
+    return {
+        "purpose": key if key in PURPOSE_MAP else "general",
+        "servers": servers,
+        "total_tools": total,
+    }
+
+
+def get_all_tools() -> Dict:
+    """Return every MCP server and its tools.
+
+    Returns:
+        Dict with servers list and total_tools count.
+    """
+    servers = [
+        {"name": name, "tools": tools}
+        for name, tools in MCP_SERVERS.items()
+    ]
+    return {"servers": servers, "total_tools": _TOTAL_TOOLS}
+
+
+def get_server_tools(server_name: str) -> List[str]:
+    """Return tool names for a single MCP server.
+
+    Args:
+        server_name: Server key (e.g. 'builder', 'supply-chain').
+
+    Returns:
+        List of tool name strings (empty if server unknown).
+    """
+    return list(MCP_SERVERS.get(server_name, []))
+
+
+def get_context_savings(purpose: str) -> Dict:
+    """Calculate context-window savings from lazy loading.
+
+    Args:
+        purpose: A purpose category key.
+
+    Returns:
+        Dict with total_tools, loaded_tools, saved_tools, savings_pct.
+    """
+    result = get_tools_for_purpose(purpose)
+    loaded = result["total_tools"]
+    saved = _TOTAL_TOOLS - loaded
+    pct = round((saved / _TOTAL_TOOLS) * 100, 1) if _TOTAL_TOOLS else 0.0
+    return {
+        "purpose": result["purpose"],
+        "total_tools": _TOTAL_TOOLS,
+        "loaded_tools": loaded,
+        "saved_tools": saved,
+        "savings_pct": pct,
+    }
+
+
+def resolve_purpose(purpose_text: str) -> str:
+    """Resolve free-form text to the best matching purpose category.
+
+    Uses simple keyword overlap scoring.  Returns 'general' when no
+    strong match is found.
+
+    Args:
+        purpose_text: Free-form description of session intent.
+
+    Returns:
+        The best matching purpose category string.
+    """
+    text = purpose_text.lower()
+    best_key = "general"
+    best_score = 0
+
+    for key, keywords in _PURPOSE_KEYWORDS.items():
+        score = sum(1 for kw in keywords if kw in text)
+        if score > best_score:
+            best_score = score
+            best_key = key
+
+    return best_key
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Lazy MCP Tool Registry — load tools by session purpose (D-ARCH-9)"
+    )
+    parser.add_argument(
+        "--purpose", metavar="KEY",
+        help="Show tools for a session purpose (e.g. compliance, build, security)",
+    )
+    parser.add_argument(
+        "--server", metavar="NAME",
+        help="Show tools for a specific MCP server",
+    )
+    parser.add_argument(
+        "--all", action="store_true",
+        help="Show all MCP servers and tools",
+    )
+    parser.add_argument(
+        "--savings", metavar="KEY",
+        help="Show context-window savings for a purpose",
+    )
+    parser.add_argument(
+        "--resolve", metavar="TEXT",
+        help="Resolve free-form text to a purpose category",
+    )
+    parser.add_argument(
+        "--json", action="store_true",
+        help="Output as JSON",
+    )
+    args = parser.parse_args()
+
+    result: Optional[Dict] = None
+
+    if args.purpose:
+        result = get_tools_for_purpose(args.purpose)
+    elif args.server:
+        tools = get_server_tools(args.server)
+        result = {"server": args.server, "tools": tools, "total_tools": len(tools)}
+    elif args.all:
+        result = get_all_tools()
+    elif args.savings:
+        result = get_context_savings(args.savings)
+    elif args.resolve:
+        matched = resolve_purpose(args.resolve)
+        result = {"input": args.resolve, "resolved_purpose": matched}
+    else:
+        parser.print_help()
+        return
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        _print_human(result, args)
+
+
+def _print_human(result: Dict, args) -> None:
+    """Pretty-print result for human consumption."""
+    if args.purpose or args.all:
+        purpose = result.get("purpose", "all")
+        print(f"Purpose: {purpose}  |  Tools loaded: {result['total_tools']}")
+        for srv in result.get("servers", []):
+            print(f"  [{srv['name']}] {', '.join(srv['tools'])}")
+    elif args.server:
+        print(f"Server: {result['server']}  |  Tools: {result['total_tools']}")
+        for t in result.get("tools", []):
+            print(f"  - {t}")
+    elif args.savings:
+        print(f"Purpose: {result['purpose']}")
+        print(f"  Total tools:  {result['total_tools']}")
+        print(f"  Loaded tools: {result['loaded_tools']}")
+        print(f"  Saved tools:  {result['saved_tools']}")
+        print(f"  Savings:      {result['savings_pct']}%")
+    elif args.resolve:
+        print(f"Input:    {result['input']}")
+        print(f"Resolved: {result['resolved_purpose']}")
+
+
+if __name__ == "__main__":
+    main()

tools/memory/__init__.py

@@ -0,0 +1,2 @@
+# [TEMPLATE: CUI // SP-CTI]
+# SPARKPILOT Memory System — Hybrid search, embeddings, session memory

tools/memory/auto_capture.py

@@ -0,0 +1,346 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Auto-capture memory from hook events and tool usage (D181-D182).
+
+Writes to memory_buffer table for async flush to memory_entries.
+Can be invoked from hook handlers or CLI.
+
+Usage:
+    python tools/memory/auto_capture.py --content "..." --source hook --tool-name scaffold --json
+    python tools/memory/auto_capture.py --flush --json
+    python tools/memory/auto_capture.py --buffer-status --json
+"""
+
+import argparse
+import hashlib
+import json
+import sqlite3
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+DB_PATH = BASE_DIR / "data" / "memory.db"
+
+VALID_SOURCES = ("hook", "manual", "thinking", "auto")
+VALID_TYPES = ("fact", "preference", "event", "insight", "task", "relationship", "thinking")
+
+
+def _load_config():
+    """Load auto-capture settings from memory_config.yaml."""
+    config_path = BASE_DIR / "args" / "memory_config.yaml"
+    defaults = {
+        "buffer_flush_threshold": 100,
+        "default_importance": 3,
+        "max_content_length": 2000,
+    }
+    try:
+        import yaml
+        if config_path.exists():
+            with open(config_path, encoding="utf-8") as f:
+                data = yaml.safe_load(f) or {}
+            ac = data.get("auto_capture", {})
+            for key in defaults:
+                if key in ac:
+                    defaults[key] = ac[key]
+    except (ImportError, Exception):
+        pass
+    return defaults
+
+
+def compute_content_hash(content):
+    """SHA-256 hash of content for deduplication (D179)."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def _get_connection(db_path=None):
+    """Get a DB connection with WAL mode for concurrent safety."""
+    path = db_path or DB_PATH
+    conn = sqlite3.connect(str(path), timeout=10)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    return conn
+
+
+def _ensure_buffer_table(conn):
+    """Create memory_buffer table if it doesn't exist."""
+    conn.executescript("""
+        CREATE TABLE IF NOT EXISTS memory_buffer (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            content TEXT NOT NULL,
+            content_hash TEXT NOT NULL,
+            type TEXT DEFAULT 'event',
+            importance INTEGER DEFAULT 3,
+            source TEXT NOT NULL DEFAULT 'hook'
+                CHECK(source IN ('hook', 'manual', 'thinking', 'auto')),
+            user_id TEXT,
+            tenant_id TEXT,
+            session_id TEXT,
+            tool_name TEXT,
+            metadata TEXT,
+            created_at TEXT DEFAULT (datetime('now'))
+        );
+        CREATE INDEX IF NOT EXISTS idx_buffer_created
+            ON memory_buffer(created_at);
+        CREATE INDEX IF NOT EXISTS idx_buffer_source
+            ON memory_buffer(source);
+        CREATE INDEX IF NOT EXISTS idx_buffer_user
+            ON memory_buffer(user_id);
+    """)
+
+
+def capture(
+    content,
+    source="hook",
+    memory_type="event",
+    importance=3,
+    user_id=None,
+    tenant_id=None,
+    session_id=None,
+    tool_name=None,
+    metadata=None,
+    db_path=None,
+):
+    """Capture content into the memory buffer.
+
+    Returns:
+        dict with status, buffer_id, and auto_flushed flag.
+    """
+    cfg = _load_config()
+    max_len = cfg.get("max_content_length", 2000)
+    if len(content) > max_len:
+        content = content[:max_len]
+
+    content_hash = compute_content_hash(content)
+    conn = _get_connection(db_path)
+    _ensure_buffer_table(conn)
+
+    try:
+        # Dedup within buffer
+        existing = conn.execute(
+            "SELECT id FROM memory_buffer WHERE content_hash = ?",
+            (content_hash,),
+        ).fetchone()
+        if existing:
+            conn.close()
+            return {"status": "duplicate", "buffer_id": existing["id"]}
+
+        conn.execute(
+            """INSERT INTO memory_buffer
+               (content, content_hash, type, importance, source,
+                user_id, tenant_id, session_id, tool_name, metadata)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+            (content, content_hash, memory_type, importance, source,
+             user_id, tenant_id, session_id, tool_name,
+             json.dumps(metadata) if metadata else None),
+        )
+        conn.commit()
+        buffer_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
+
+        # Check buffer size for auto-flush threshold
+        count = conn.execute("SELECT COUNT(*) FROM memory_buffer").fetchone()[0]
+        conn.close()
+
+        result = {
+            "status": "captured",
+            "buffer_id": buffer_id,
+            "content_hash": content_hash[:16],
+            "buffer_size": count,
+        }
+
+        threshold = cfg.get("buffer_flush_threshold", 100)
+        if count >= threshold:
+            flush_result = flush_buffer(db_path=db_path)
+            result["auto_flushed"] = True
+            result["flush_result"] = flush_result
+
+        return result
+
+    except Exception as exc:
+        conn.close()
+        return {"status": "error", "error": str(exc)}
+
+
+def flush_buffer(db_path=None):
+    """Flush buffer entries to memory_entries with dedup check.
+
+    Returns:
+        dict with flushed, duplicates, and errors counts.
+    """
+    conn = _get_connection(db_path)
+    _ensure_buffer_table(conn)
+
+    try:
+        rows = conn.execute(
+            "SELECT id, content, content_hash, type, importance, source, "
+            "user_id, tenant_id, session_id, tool_name, metadata, created_at "
+            "FROM memory_buffer ORDER BY created_at ASC"
+        ).fetchall()
+
+        if not rows:
+            conn.close()
+            return {"flushed": 0, "duplicates": 0, "errors": 0}
+
+        flushed = 0
+        duplicates = 0
+        errors = 0
+        flushed_ids = []
+
+        for row in rows:
+            try:
+                # Check dedup against memory_entries
+                if row["user_id"]:
+                    existing = conn.execute(
+                        "SELECT id FROM memory_entries WHERE content_hash = ? AND user_id = ?",
+                        (row["content_hash"], row["user_id"]),
+                    ).fetchone()
+                else:
+                    existing = conn.execute(
+                        "SELECT id FROM memory_entries WHERE content_hash = ? AND user_id IS NULL",
+                        (row["content_hash"],),
+                    ).fetchone()
+
+                if existing:
+                    duplicates += 1
+                    flushed_ids.append(row["id"])
+                    continue
+
+                conn.execute(
+                    """INSERT INTO memory_entries
+                       (content, type, importance, content_hash,
+                        user_id, tenant_id, source)
+                       VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                    (row["content"], row["type"], row["importance"],
+                     row["content_hash"], row["user_id"], row["tenant_id"],
+                     row["source"]),
+                )
+                flushed += 1
+                flushed_ids.append(row["id"])
+
+            except Exception:
+                errors += 1
+
+        # Remove flushed entries from buffer
+        if flushed_ids:
+            placeholders = ",".join("?" * len(flushed_ids))
+            conn.execute(
+                f"DELETE FROM memory_buffer WHERE id IN ({placeholders})",
+                flushed_ids,
+            )
+
+        conn.commit()
+        conn.close()
+
+        return {
+            "flushed": flushed,
+            "duplicates": duplicates,
+            "errors": errors,
+            "total_processed": len(rows),
+        }
+
+    except Exception as exc:
+        conn.close()
+        return {"flushed": 0, "errors": 1, "error": str(exc)}
+
+
+def buffer_status(db_path=None):
+    """Return buffer statistics."""
+    cfg = _load_config()
+    conn = _get_connection(db_path)
+    _ensure_buffer_table(conn)
+
+    try:
+        total = conn.execute("SELECT COUNT(*) FROM memory_buffer").fetchone()[0]
+        by_source = conn.execute(
+            "SELECT source, COUNT(*) as cnt FROM memory_buffer GROUP BY source"
+        ).fetchall()
+        oldest = conn.execute(
+            "SELECT MIN(created_at) FROM memory_buffer"
+        ).fetchone()[0]
+        conn.close()
+
+        return {
+            "classification": "CUI // SP-CTI",
+            "total_buffered": total,
+            "by_source": {row["source"]: row["cnt"] for row in by_source},
+            "oldest_entry": oldest,
+            "flush_threshold": cfg.get("buffer_flush_threshold", 100),
+        }
+    except Exception as exc:
+        conn.close()
+        return {"total_buffered": 0, "error": str(exc)}
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Auto-capture memory from hooks/tools (D181)"
+    )
+    parser.add_argument("--content", help="Content to capture")
+    parser.add_argument("--source", choices=VALID_SOURCES, default="hook",
+                        help="Capture source")
+    parser.add_argument("--type", choices=VALID_TYPES, default="event",
+                        dest="memory_type", help="Memory type")
+    parser.add_argument("--importance", type=int, default=3, help="Importance 1-10")
+    parser.add_argument("--user-id", help="User ID (D180)")
+    parser.add_argument("--tenant-id", help="Tenant ID (D180)")
+    parser.add_argument("--session-id", help="Session/correlation ID")
+    parser.add_argument("--tool-name", help="Tool name for context")
+    parser.add_argument("--flush", action="store_true", help="Flush buffer to memory_entries")
+    parser.add_argument("--buffer-status", action="store_true", help="Show buffer stats")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    if args.buffer_status:
+        result = buffer_status()
+        if args.json:
+            print(json.dumps(result, indent=2))
+        else:
+            print(f"Buffered: {result['total_buffered']} entries")
+            for src, cnt in result.get("by_source", {}).items():
+                print(f"  {src}: {cnt}")
+            if result.get("oldest_entry"):
+                print(f"  Oldest: {result['oldest_entry']}")
+            print(f"  Flush threshold: {result['flush_threshold']}")
+        return
+
+    if args.flush:
+        result = flush_buffer()
+        if args.json:
+            print(json.dumps({"classification": "CUI // SP-CTI", **result}, indent=2))
+        else:
+            print(f"Flushed {result['flushed']} entries "
+                  f"({result['duplicates']} duplicates, {result['errors']} errors)")
+        return
+
+    if not args.content:
+        print("Error: --content required (or use --flush / --buffer-status)")
+        sys.exit(1)
+
+    result = capture(
+        content=args.content,
+        source=args.source,
+        memory_type=args.memory_type,
+        importance=args.importance,
+        user_id=args.user_id,
+        tenant_id=args.tenant_id,
+        session_id=args.session_id,
+        tool_name=args.tool_name,
+    )
+    if args.json:
+        print(json.dumps({"classification": "CUI // SP-CTI", **result}, indent=2))
+    else:
+        if result["status"] == "captured":
+            print(f"Captured to buffer (id: {result['buffer_id']}, "
+                  f"buffer size: {result['buffer_size']})")
+            if result.get("auto_flushed"):
+                fr = result["flush_result"]
+                print(f"  Auto-flushed: {fr['flushed']} entries")
+        elif result["status"] == "duplicate":
+            print(f"Duplicate (existing buffer id: {result['buffer_id']})")
+        else:
+            print(f"Error: {result.get('error', 'unknown')}")
+
+
+if __name__ == "__main__":
+    main()