agent-corex 1.0.0__py3-none-any.whl

agent_core/__init__.py

@@ -0,0 +1,19 @@
+"""
+Agent-Core: Fast, accurate MCP tool retrieval engine for LLMs with semantic search.
+
+Provides multiple ranking methods for selecting the most relevant tools from large sets.
+"""
+
+__version__ = "1.0.0"
+__author__ = "Ankit Agarwal"
+__email__ = "ankitagarwalpro@gmail.com"
+
+from agent_core.retrieval.ranker import rank_tools
+from agent_core.tools.registry import ToolRegistry
+from agent_core.tools.mcp.mcp_loader import MCPLoader
+
+__all__ = [
+    "rank_tools",
+    "ToolRegistry",
+    "MCPLoader",
+]

agent_core/api/__init__.py

@@ -0,0 +1 @@
+"""API module."""

agent_core/api/main.py

@@ -0,0 +1,46 @@
+from fastapi import FastAPI
+from packages.tools.registry import ToolRegistry
+from packages.retrieval.ranker import rank_tools
+
+app = FastAPI()
+
+tool_registry = ToolRegistry()
+
+# Example tools (replace with MCP-loaded later)
+tool_registry.register({
+    "name": "edit_file",
+    "description": "Edit a file with line-based changes"
+})
+
+tool_registry.register({
+    "name": "write_file",
+    "description": "Create or overwrite a file"
+})
+
+tool_registry.register({
+    "name": "run_tests",
+    "description": "Run test suite"
+})
+
+
+@app.get("/retrieve_tools")
+def retrieve_tools(query: str, top_k: int = 5, method: str = "hybrid"):
+    """
+    Retrieve the most relevant tools for a given query.
+
+    Args:
+        query: Search query describing what you need
+        top_k: Number of results to return (default: 5)
+        method: Ranking method - 'keyword', 'hybrid', or 'embedding' (default: 'hybrid')
+
+    Returns:
+        List of tools ranked by relevance
+    """
+    tools = tool_registry.get_all_tools()
+    ranked = rank_tools(query, tools, top_k, method=method)
+    return ranked
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}

agent_core/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI module for Agent-Core."""

agent_core/cli/main.py

@@ -0,0 +1,164 @@
+"""
+Agent-Core CLI interface.
+
+Provides command-line tools for managing and using Agent-Core.
+"""
+
+import typer
+from typing import Optional
+import json
+
+app = typer.Typer(
+    name="agent-corex",
+    help="Fast, accurate MCP tool retrieval engine for LLMs",
+    no_args_is_help=True
+)
+
+
+@app.command()
+def retrieve(
+    query: str = typer.Argument(..., help="Search query for tool retrieval"),
+    top_k: int = typer.Option(5, "--top-k", "-k", help="Number of results to return"),
+    method: str = typer.Option("hybrid", "--method", "-m", help="Ranking method: keyword, hybrid, or embedding"),
+    config: Optional[str] = typer.Option(None, "--config", "-c", help="Path to mcp.json config file")
+):
+    """
+    Retrieve the most relevant tools for a given query.
+
+    Example:
+        agent-corex retrieve "edit a file" --top-k 5 --method hybrid
+    """
+    try:
+        from agent_core.retrieval.ranker import rank_tools
+        from agent_core.tools.registry import ToolRegistry
+        from agent_core.tools.mcp.mcp_loader import MCPLoader
+        import pathlib
+
+        # Load tools
+        registry = ToolRegistry()
+
+        # Try to load MCP servers if config provided
+        if config:
+            config_path = pathlib.Path(config)
+            if config_path.exists():
+                try:
+                    loader = MCPLoader(str(config_path))
+                    manager = loader.load()
+                    tools = manager.get_all_tools()
+                except Exception as e:
+                    typer.echo(f"Warning: Failed to load MCP servers: {e}", err=True)
+                    tools = registry.get_all_tools()
+            else:
+                typer.echo(f"Config file not found: {config}", err=True)
+                tools = registry.get_all_tools()
+        else:
+            tools = registry.get_all_tools()
+
+        if not tools:
+            typer.echo("No tools available", err=True)
+            raise typer.Exit(1)
+
+        # Retrieve tools
+        results = rank_tools(query, tools, top_k=top_k, method=method)
+
+        if not results:
+            typer.echo(f"No tools found for query: {query}")
+            raise typer.Exit(0)
+
+        # Display results
+        typer.echo(f"\nFound {len(results)} tool(s) for: '{query}'\n")
+        for i, tool in enumerate(results, 1):
+            typer.echo(f"{i}. {tool['name']}")
+            typer.echo(f"   {tool.get('description', 'No description')}\n")
+
+    except Exception as e:
+        typer.echo(f"Error: {str(e)}", err=True)
+        raise typer.Exit(1)
+
+
+@app.command()
+def start(
+    host: str = typer.Option("127.0.0.1", "--host", "-h", help="Server host"),
+    port: int = typer.Option(8000, "--port", "-p", help="Server port"),
+    reload: bool = typer.Option(True, "--reload/--no-reload", help="Enable auto-reload"),
+    config: Optional[str] = typer.Option(None, "--config", "-c", help="Path to mcp.json config file")
+):
+    """
+    Start the Agent-Core API server.
+
+    Example:
+        agent-corex start --host 0.0.0.0 --port 8000
+    """
+    import uvicorn
+    import os
+
+    # Set config path if provided
+    if config:
+        os.environ["AGENT_CORE_CONFIG"] = config
+
+    typer.echo(f"Starting Agent-Core API server at http://{host}:{port}")
+    typer.echo("Press Ctrl+C to stop\n")
+
+    uvicorn.run(
+        "agent_core.api.main:app",
+        host=host,
+        port=port,
+        reload=reload,
+        log_level="info"
+    )
+
+
+@app.command()
+def version():
+    """Show Agent-Core version."""
+    from agent_core import __version__
+    typer.echo(f"Agent-Core {__version__}")
+
+
+@app.command()
+def health():
+    """Check API health (requires running server)."""
+    import requests
+    try:
+        response = requests.get("http://127.0.0.1:8000/health", timeout=5)
+        if response.status_code == 200:
+            typer.echo("✓ Agent-Core API is healthy")
+        else:
+            typer.echo(f"✗ Agent-Core API returned status {response.status_code}", err=True)
+    except requests.exceptions.ConnectionError:
+        typer.echo("✗ Cannot connect to Agent-Core API. Is it running?", err=True)
+        raise typer.Exit(1)
+    except Exception as e:
+        typer.echo(f"✗ Error: {str(e)}", err=True)
+        raise typer.Exit(1)
+
+
+@app.command()
+def config():
+    """Show configuration information."""
+    import pathlib
+    from agent_core import __version__
+
+    typer.echo(f"Agent-Core {__version__}\n")
+    typer.echo("Configuration:")
+    typer.echo(f"  Python version: {__import__('sys').version.split()[0]}")
+    typer.echo(f"  Installation path: {pathlib.Path(__import__('agent_core').__file__).parent}")
+
+    # Check if dependencies are installed
+    deps = {
+        "fastapi": "FastAPI",
+        "sentence_transformers": "Sentence Transformers",
+        "faiss": "FAISS",
+    }
+
+    typer.echo("\nDependencies:")
+    for module, name in deps.items():
+        try:
+            __import__(module)
+            typer.echo(f"  ✓ {name}")
+        except ImportError:
+            typer.echo(f"  ✗ {name} (not installed)")
+
+
+if __name__ == "__main__":
+    app()

agent_core/retrieval/embeddings.py

@@ -0,0 +1,118 @@
+"""
+Embedding-based tool indexing and semantic search using FAISS and sentence-transformers.
+
+This module provides semantic search capabilities for tools using pre-trained embeddings.
+Tools are indexed by their name and description, enabling semantic similarity matching
+for queries that aren't exact keyword matches.
+"""
+
+from sentence_transformers import SentenceTransformer
+import faiss
+import numpy as np
+from typing import List, Dict, Any, Optional
+
+
+class EmbeddingIndexer:
+    """
+    FAISS-based semantic search indexer for tools.
+
+    Uses sentence-transformers to generate embeddings for tool names and descriptions,
+    then uses FAISS for efficient similarity search.
+
+    Attributes:
+        model: Cached SentenceTransformer model (shared across instances)
+        tools: List of tools in the index
+        index: FAISS index for similarity search
+    """
+
+    _model = None  # Class-level cache to avoid reloading model
+
+    def __init__(self, tools: Optional[List[Dict[str, Any]]] = None):
+        """
+        Initialize the embedding indexer.
+
+        Args:
+            tools: Optional list of tools to index. Each tool should have 'name' and 'description'.
+        """
+        # Load model once and cache at class level
+        if EmbeddingIndexer._model is None:
+            EmbeddingIndexer._model = SentenceTransformer(
+                "sentence-transformers/all-MiniLM-L6-v2",
+                cache_folder=".agent_core_models"
+            )
+
+        self.model = EmbeddingIndexer._model
+        self.tools = tools or []
+        self.index = None
+
+        if self.tools:
+            self._build_index(self.tools)
+
+    def _build_index(self, tools: List[Dict[str, Any]]) -> None:
+        """
+        Build FAISS index from tools.
+
+        Args:
+            tools: List of tools to index.
+        """
+        texts = [
+            f"{t['name']} {t.get('description', '')}"
+            for t in tools
+        ]
+
+        embeddings = self.model.encode(texts)
+        dim = embeddings.shape[1]
+
+        self.index = faiss.IndexFlatL2(dim)
+        self.index.add(np.array(embeddings, dtype=np.float32))
+
+    def add_tools(self, tools: List[Dict[str, Any]]) -> None:
+        """
+        Add new tools to the index.
+
+        Args:
+            tools: List of tools to add.
+        """
+        texts = [
+            f"{t['name']} {t.get('description', '')}"
+            for t in tools
+        ]
+
+        embeddings = self.model.encode(texts)
+
+        if self.index is None:
+            dim = embeddings.shape[1]
+            self.index = faiss.IndexFlatL2(dim)
+
+        self.index.add(np.array(embeddings, dtype=np.float32))
+
+        for tool in tools:
+            self.tools.append(tool)
+
+    def search(self, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        """
+        Search for tools semantically similar to the query.
+
+        Args:
+            query: The search query.
+            top_k: Number of top results to return.
+
+        Returns:
+            List of tools sorted by semantic similarity to the query.
+        """
+        if top_k is None:
+            top_k = 5
+
+        if self.index is None or not self.tools:
+            return []
+
+        query_vec = self.model.encode([query])
+        D, I = self.index.search(np.array(query_vec, dtype=np.float32), min(top_k, len(self.tools)))
+
+        results = [
+            self.tools[i]
+            for i in I[0]
+            if 0 <= i < len(self.tools)
+        ]
+
+        return results

agent_core/retrieval/hybrid_scorer.py

@@ -0,0 +1,105 @@
+"""
+Hybrid scoring system combining keyword-based and embedding-based relevance.
+
+This module provides scoring methods that combine:
+1. Keyword overlap (fast, exact matching)
+2. Semantic similarity via embeddings (catches related tools)
+
+The hybrid approach gives the best of both worlds: speed + accuracy.
+"""
+
+from typing import List, Dict, Any, Tuple, Optional
+from .scorer import score as keyword_score
+from .embeddings import EmbeddingIndexer
+from sentence_transformers import SentenceTransformer
+import numpy as np
+
+
+class HybridScorer:
+    """
+    Combines keyword-based and embedding-based scoring.
+
+    Uses a weighted combination of:
+    - Keyword overlap score (0.3 weight)
+    - Embedding similarity score (0.7 weight)
+
+    This gives semantic search the priority while still considering exact keyword matches.
+    """
+
+    def __init__(self, keyword_weight: float = 0.3, embedding_weight: float = 0.7):
+        """
+        Initialize the hybrid scorer.
+
+        Args:
+            keyword_weight: Weight for keyword-based scoring (0-1).
+            embedding_weight: Weight for embedding-based scoring (0-1).
+        """
+        self.keyword_weight = keyword_weight
+        self.embedding_weight = embedding_weight
+
+        # Normalize weights
+        total = keyword_weight + embedding_weight
+        self.keyword_weight /= total
+        self.embedding_weight /= total
+
+        # Initialize the embedding model
+        self.model = SentenceTransformer(
+            "sentence-transformers/all-MiniLM-L6-v2",
+            cache_folder=".agent_core_models"
+        )
+
+    def score(self, query: str, tool: Dict[str, Any]) -> float:
+        """
+        Compute hybrid score for a tool.
+
+        Args:
+            query: The search query.
+            tool: Tool dictionary with 'name' and 'description'.
+
+        Returns:
+            Hybrid score between 0 and 1.
+        """
+        # Get keyword score
+        kw_score = keyword_score(query, tool)
+
+        # Get embedding score
+        query_vec = self.model.encode([query])
+        tool_text = f"{tool['name']} {tool.get('description', '')}"
+        tool_vec = self.model.encode([tool_text])
+
+        # Cosine similarity (convert L2 distance to similarity)
+        # FAISS uses L2 distance, so we convert: similarity = 1 / (1 + distance)
+        similarity = np.dot(query_vec[0], tool_vec[0]) / (
+            np.linalg.norm(query_vec[0]) * np.linalg.norm(tool_vec[0])
+        )
+        # Normalize to 0-1 range
+        embedding_score = (similarity + 1) / 2
+
+        # Combine scores
+        hybrid_score = (
+            self.keyword_weight * kw_score +
+            self.embedding_weight * embedding_score
+        )
+
+        return hybrid_score
+
+    def score_batch(
+        self, query: str, tools: List[Dict[str, Any]]
+    ) -> List[Tuple[Dict[str, Any], float]]:
+        """
+        Score multiple tools efficiently.
+
+        Args:
+            query: The search query.
+            tools: List of tools to score.
+
+        Returns:
+            List of (tool, score) tuples.
+        """
+        scored = []
+
+        for tool in tools:
+            s = self.score(query, tool)
+            scored.append((tool, s))
+
+        return scored

agent_core/retrieval/ranker.py

@@ -0,0 +1,96 @@
+"""
+Tool ranking engine for retrieval.
+
+Supports multiple ranking strategies:
+1. Keyword-only (fast, for simple queries)
+2. Hybrid (keyword + embeddings, recommended)
+3. Embedding-only (semantic search only)
+"""
+
+from typing import List, Dict, Any, Optional
+from .scorer import score as keyword_score
+from .hybrid_scorer import HybridScorer
+
+
+def rank_tools(
+    query: str,
+    tools: List[Dict[str, Any]],
+    top_k: int = 5,
+    method: str = "hybrid"
+) -> List[Dict[str, Any]]:
+    """
+    Rank tools by relevance to a query.
+
+    Args:
+        query: The search query.
+        tools: List of tools to rank.
+        top_k: Number of top results to return.
+        method: Ranking method ('keyword', 'hybrid', or 'embedding').
+               - 'keyword': Fast, exact match only
+               - 'hybrid': Recommended, combines keyword + semantic
+               - 'embedding': Semantic similarity only
+
+    Returns:
+        Top-k tools ranked by relevance.
+    """
+    if method == "keyword":
+        return _rank_by_keyword(query, tools, top_k)
+    elif method == "hybrid":
+        return _rank_by_hybrid(query, tools, top_k)
+    elif method == "embedding":
+        return _rank_by_embedding(query, tools, top_k)
+    else:
+        raise ValueError(f"Unknown ranking method: {method}")
+
+
+def _rank_by_keyword(
+    query: str, tools: List[Dict[str, Any]], top_k: int
+) -> List[Dict[str, Any]]:
+    """Rank by keyword overlap only (fast)."""
+    scored = []
+
+    for tool in tools:
+        s = keyword_score(query, tool)
+        scored.append((tool, s))
+
+    # Sort descending
+    scored.sort(key=lambda x: x[1], reverse=True)
+
+    # Filter zero scores
+    ranked = [tool for tool, s in scored if s > 0]
+
+    return ranked[:top_k]
+
+
+def _rank_by_hybrid(
+    query: str, tools: List[Dict[str, Any]], top_k: int
+) -> List[Dict[str, Any]]:
+    """Rank by hybrid scoring (keyword + embeddings)."""
+    try:
+        scorer = HybridScorer()
+        scored = scorer.score_batch(query, tools)
+
+        # Sort descending
+        scored.sort(key=lambda x: x[1], reverse=True)
+
+        # Return top-k (don't filter zero scores for embeddings)
+        return [tool for tool, s in scored[:top_k]]
+    except Exception as e:
+        # Fallback to keyword-only if embedding model fails
+        print(f"Warning: Embedding model failed ({e}), falling back to keyword ranking")
+        return _rank_by_keyword(query, tools, top_k)
+
+
+def _rank_by_embedding(
+    query: str, tools: List[Dict[str, Any]], top_k: int
+) -> List[Dict[str, Any]]:
+    """Rank by semantic similarity only (requires embeddings)."""
+    from .embeddings import EmbeddingIndexer
+
+    try:
+        indexer = EmbeddingIndexer(tools)
+        return indexer.search(query, top_k)
+    except Exception as e:
+        # Fallback to keyword-only if embedding model fails
+        print(f"Warning: Embedding model failed ({e}), falling back to keyword ranking")
+        return _rank_by_keyword(query, tools, top_k)

agent_core/retrieval/scorer.py

@@ -0,0 +1,18 @@
+import re
+
+def tokenize(text: str):
+    return set(re.findall(r"\w+", text.lower()))
+
+def score(query: str, tool) -> float:
+    """
+    Basic keyword overlap scoring
+    """
+    query_tokens = tokenize(query)
+    text = f"{tool['name']} {tool.get('description', '')}"
+    tool_tokens = tokenize(text)
+
+    if not tool_tokens:
+        return 0.0
+
+    overlap = query_tokens.intersection(tool_tokens)
+    return len(overlap) / len(query_tokens) if query_tokens else 0.0

agent_core/tools/base_tool.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+
+
+class BaseTool(ABC):
+
+    name = ""
+
+    @abstractmethod
+    async def run(self, **kwargs):
+        pass