gnosys-strata 1.1.4__py3-none-any.whl

strata/utils/field_search.py

@@ -0,0 +1,233 @@
+"""
+Field-based weighted search engine
+
+A simpler alternative to BM25 that focuses on field-level matching
+with explicit weights and avoids document length bias.
+
+## Algorithm Design
+
+This search engine implements a three-layer scoring system to prevent score explosion
+and ensure relevant results:
+
+### Layer 1: Match Quality Scoring
+For each token-field match, we assign a base score based on match quality:
+- Exact match: weight × 3.0 (e.g., field value "projects" == query "projects")
+- Word boundary match: weight × 2.0 (e.g., "list projects" matches query "projects" as complete word)
+- Partial match: weight × 1.0 (e.g., "project_list" contains query "project")
+
+### Layer 2: Intra-field Token Decay (Harmonic Series)
+When multiple query tokens match within the same field, we apply diminishing returns:
+- 1st token: 100% of score
+- 2nd token: 50% of score (1/2)
+- 3rd token: 33% of score (1/3)
+- 4th token: 25% of score (1/4)
+
+This prevents long descriptions from accumulating excessive scores by matching many tokens.
+
+### Layer 3: Per-field Logarithmic Dampening
+After calculating field scores, we apply logarithmic dampening based on field type:
+- Description fields (description, param_desc): log(1 + score) × 5 (stronger dampening)
+- Identifier fields (service, operation, tag, path, etc.): log(1 + score) × 10 (lighter dampening)
+
+This prevents any single field from dominating the final score, especially verbose fields.
+
+### Final Score Calculation
+- Sum all dampened field scores
+- Add diversity bonus: sqrt(matched_field_types) × 3
+  (rewards matching across multiple field types)
+
+## Problem Scenarios This Solves
+
+### Scenario 1: Keyword Repetition
+Query: "projects"
+Without dampening:
+- Endpoint A: service="projects"(90) + tag="projects"(90) + path="/projects"(60) +
+              description="manage projects"(20) = 260 points
+- Endpoint B: service="users"(0) + operation="get_user_projects"(60) = 60 points
+
+With our algorithm:
+- Endpoint A: log(91)×10 + log(91)×10 + log(61)×10 + log(21)×5 = 45.5+45.5+40.2+15.2 = 146.4
+- Endpoint B: log(61)×10 = 40.2
+
+Still favors A but with reasonable margin, not 4x difference.
+
+### Scenario 2: Long Description Domination
+Query: "create user project pipeline"
+Without dampening:
+- Endpoint A: description contains all 4 words = 20×4 = 80 points
+- Endpoint B: operation="create_pipeline" = 30×2 = 60 points
+
+With our algorithm:
+- Endpoint A: (20 + 20/2 + 20/3 + 20/4) = 41.7 → log(42.7)×5 = 18.6 points
+- Endpoint B: 30×2 = 60 → log(61)×10 = 40.2 points
+
+Now B correctly ranks higher as it's more specific.
+
+### Scenario 3: Exact Service Name Match
+Query: "projects"
+- Service name exactly "projects": 30×3=90 → log(91)×10 = 45.5 points
+This ensures exact matches still get high scores despite dampening.
+
+## Weight Configuration
+
+Weights should be configured based on field importance:
+- High (30): service, operation, tag, path - core identifiers
+- Medium (20): summary, description - contextual information
+- Low (5): method, param - auxiliary information
+- Minimal (1-2): param_desc, body_field - verbose/detailed fields
+
+The weights are passed during document indexing, allowing different OpenAPI
+implementations to customize based on their documentation structure.
+"""
+
+import math
+import re
+from typing import List, Tuple
+
+
+class FieldSearchEngine:
+    """
+    Simple field-based search engine with weighted scoring
+    Compatible with BM25SearchEngine interface
+    """
+
+    def __init__(self, **kwargs):
+        """Initialize the search engine (kwargs for compatibility with BM25SearchEngine)"""
+        self.documents = []
+        self.corpus_metadata = None
+
+    def build_index(self, documents: List[Tuple[List[Tuple[str, str, int]], str]]):
+        """
+        Build index from documents
+
+        Args:
+            documents: List of (fields, doc_id) tuples
+                      fields: List of (field_key, field_value, weight) tuples
+                              weight is used as field priority
+                      doc_id: Document identifier string
+        """
+        self.documents = []
+        self.corpus_metadata = []
+
+        for fields, doc_id in documents:
+            # Store document with structured fields and their weights
+            doc_fields = {}
+            field_weights = {}
+
+            for field_key, field_value, weight in fields:
+                if field_value:
+                    # Group values by field type
+                    if field_key not in doc_fields:
+                        doc_fields[field_key] = []
+                        field_weights[field_key] = weight
+                    doc_fields[field_key].append(field_value.lower())
+
+                    # Use the highest weight if multiple values for same field
+                    if weight > field_weights.get(field_key, 0):
+                        field_weights[field_key] = weight
+
+            self.documents.append(
+                {"id": doc_id, "fields": doc_fields, "weights": field_weights}
+            )
+            self.corpus_metadata.append(doc_id)
+
+    def search(self, query: str, top_k: int = 10) -> List[Tuple[float, str]]:
+        """
+        Search documents with field-weighted scoring and logarithmic dampening
+
+        Args:
+            query: Search query string
+            top_k: Number of top results to return
+
+        Returns:
+            List of (score, doc_id) tuples sorted by score descending
+        """
+        if not self.documents:
+            return []
+
+        # Tokenize query into words
+        query_tokens = query.lower().split()
+
+        results = []
+
+        for doc in self.documents:
+            # Track scores by field type to apply per-field dampening
+            field_scores = {}
+            matched_field_types = set()
+
+            # Check each field type
+            for field_type, field_values in doc["fields"].items():
+                # Get weight from document's field weights
+                field_weight = doc["weights"].get(field_type, 1.0)
+
+                # Track tokens matched in this field
+                field_token_scores = []
+                matched_tokens = set()
+
+                # For each query token
+                for token in query_tokens:
+                    # Check if token appears in any value of this field
+                    best_match_score = 0
+
+                    for value in field_values:
+                        if (
+                            self._match_token(token, value)
+                            and token not in matched_tokens
+                        ):
+                            # Calculate match quality
+                            match_score = 0
+
+                            # Exact match gets highest score
+                            if value == token:
+                                match_score = 3.0
+                            # Word boundary match (complete word)
+                            elif re.search(r"\b" + re.escape(token) + r"\b", value):
+                                match_score = 2.0
+                            # Partial match gets base score
+                            else:
+                                match_score = 1.0
+
+                            best_match_score = max(best_match_score, match_score)
+
+                    if best_match_score > 0:
+                        matched_tokens.add(token)
+                        field_token_scores.append(field_weight * best_match_score)
+                        matched_field_types.add(field_type)
+
+                # Apply diminishing returns for multiple tokens in same field
+                if field_token_scores:
+                    # Sort scores in descending order
+                    field_token_scores.sort(reverse=True)
+
+                    # Apply decay: 1st token 100%, 2nd 50%, 3rd 33%, etc.
+                    field_total = 0
+                    for i, token_score in enumerate(field_token_scores):
+                        field_total += token_score / (i + 1)
+
+                    # Apply logarithmic dampening per field to prevent single field domination
+                    # This prevents description or other verbose fields from dominating
+                    if field_type in ["description", "param_desc"]:
+                        # Stronger dampening for description fields
+                        field_scores[field_type] = math.log(1 + field_total) * 5
+                    else:
+                        # Lighter dampening for identifier fields
+                        field_scores[field_type] = math.log(1 + field_total) * 10
+
+            # Calculate final score
+            if field_scores:
+                # Sum all field scores (already dampened per field)
+                total_score = sum(field_scores.values())
+
+                # Add diversity bonus for matching multiple field types
+                diversity_bonus = math.sqrt(len(matched_field_types)) * 3
+
+                final_score = total_score + diversity_bonus
+                results.append((final_score, doc["id"]))
+
+        # Sort by score descending and return top k
+        results.sort(key=lambda x: x[0], reverse=True)
+        return results[:top_k]
+
+    def _match_token(self, token: str, text: str) -> bool:
+        """Check if token matches in text"""
+        return token in text

strata/utils/shared_search.py

@@ -0,0 +1,202 @@
+"""
+Shared search utility for both single_server and strata_server.
+Provides type-safe interfaces for searching through MCP tools
+Uses a unified generic approach to reduce code duplication.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from mcp import types
+
+from strata.utils.bm25_search import BM25SearchEngine
+
+
+class UniversalToolSearcher:
+    """
+    Universal searcher that handles all tool types
+    using a single unified approach based on function names.
+    """
+
+    def __init__(self, mixed_tools_map: Dict[str, List[Any]]):
+        """
+        Initialize universal searcher with mixed tool types.
+
+        Args:
+            mixed_tools_map: Dictionary mapping categories to tools.
+                           Tools can be either types.Tool objects or dict objects.
+        """
+        self.tools_map = mixed_tools_map
+        self.search_engine = self._build_index()
+
+    def _get_tool_name(self, tool: Any) -> Optional[str]:
+        """Extract name from any tool type."""
+        if isinstance(tool, types.Tool):
+            return tool.name if tool.name else None
+        elif isinstance(tool, dict):
+            return tool.get("name")
+        return None
+
+    def _get_tool_field(self, tool: Any, field_name: str, default: Any = None) -> Any:
+        """Extract field value from any tool type."""
+        if isinstance(tool, types.Tool):
+            return getattr(tool, field_name, default)
+        elif isinstance(tool, dict):
+            return tool.get(field_name, default)
+        return default
+
+    def _build_index(self) -> BM25SearchEngine:
+        """Build unified search index from all tools."""
+        documents = []
+
+        for category_name, tools in self.tools_map.items():
+            for tool in tools:
+                # Get tool name (function name)
+                tool_name = self._get_tool_name(tool)
+                if not tool_name:
+                    continue
+
+                # Build weighted fields
+                fields = []
+
+                # Core identifiers - highest weight
+                fields.append(("category", category_name.lower(), 30))
+                fields.append(("operation", tool_name.lower(), 30))
+
+                # Title if available
+                title = self._get_tool_field(tool, "title", "")
+                if title:
+                    fields.append(("title", str(title).lower(), 30))
+
+                # Description/Summary - highest weight
+                description = self._get_tool_field(tool, "description", "")
+                if description:
+                    fields.append(("description", str(description).lower(), 30))
+
+                summary = self._get_tool_field(tool, "summary", "")
+                if summary:
+                    fields.append(("summary", str(summary).lower(), 30))
+
+                tags = self._get_tool_field(tool, "tags", [])
+                if isinstance(tags, list):
+                    for tag in tags:
+                        if tag:
+                            fields.append(("tag", str(tag).lower(), 30))
+
+                path = self._get_tool_field(tool, "path", "")
+                if path:
+                    fields.append(("path", str(path).lower(), 30))
+
+                method = self._get_tool_field(tool, "method", "")
+                if method:
+                    fields.append(("method", str(method).lower(), 15))
+
+                for param_type in ["path_params", "query_params"]:
+                    params = self._get_tool_field(tool, param_type, {})
+                    for param_name, param_info in params.items():
+                        fields.append(
+                            (f"{param_type}/{param_name}", param_name.lower(), 15)
+                        )
+                        if isinstance(param_info, dict):
+                            param_desc = param_info.get("description", "")
+                            if param_desc:
+                                fields.append(
+                                    (
+                                        f"{param_type}/{param_name}_desc",
+                                        param_desc.lower(),
+                                        15,
+                                    )
+                                )
+
+                # Body schema fields
+                body_schema = self._get_tool_field(tool, "body_schema", {})
+                for param_name, param_info in body_schema.get("properties", {}).items():
+                    fields.append((f"body_schema/{param_name}", param_name.lower(), 15))
+                    if isinstance(param_info, dict):
+                        param_desc = param_info.get("description", "")
+                        if param_desc:
+                            fields.append(
+                                (
+                                    f"body_schema/{param_name}_desc",
+                                    param_desc.lower(),
+                                    15,
+                                )
+                            )
+
+                response_schema = self._get_tool_field(tool, "response_schema", {})
+                for param_name, param_info in response_schema.get(
+                    "properties", {}
+                ).items():
+                    fields.append(
+                        (f"response_schema/{param_name}", param_name.lower(), 5)
+                    )
+                    if isinstance(param_info, dict):
+                        param_desc = param_info.get("description", "")
+                        if param_desc:
+                            fields.append(
+                                (
+                                    f"response_schema/{param_name}_desc",
+                                    param_desc.lower(),
+                                    5,
+                                )
+                            )
+
+                # Create document ID
+                doc_id = f"{category_name}::{tool_name}"
+                if fields:
+                    documents.append((fields, doc_id))
+
+        # Build search index
+        search_engine = BM25SearchEngine()
+        search_engine.build_index(documents)
+        return search_engine
+
+    def search(self, query: str, max_results: int = 10) -> List[Dict[str, Any]]:
+        """
+        Search through all tools.
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of results to return
+
+        Returns:
+            List of search results with tool information
+        """
+        if self.search_engine is None:
+            return []
+
+        # Perform search
+        search_results = self.search_engine.search(query.lower(), top_k=max_results)
+
+        # Build results
+        results = []
+
+        for score, doc_id in search_results:
+            # Parse doc_id
+            if "::" not in doc_id:
+                continue
+
+            category_name, tool_name = doc_id.split("::", 1)
+
+            # Find the tool
+            if category_name in self.tools_map:
+                for tool in self.tools_map[category_name]:
+                    if self._get_tool_name(tool) == tool_name:
+                        result = {
+                            "name": tool_name,
+                            "description": self._get_tool_field(
+                                tool, "description", ""
+                            ),
+                            "category_name": category_name,
+                            # "relevance_score": score,
+                        }
+
+                        # Add optional fields if they exist
+                        for field in ["title", "summary"]:
+                            value = self._get_tool_field(tool, field)
+                            if value:
+                                result[field] = value
+
+                        results.append(result)
+                        break
+
+        return results

strata/utils/tool_integration.py

@@ -0,0 +1,269 @@
+"""Tool integration utilities for adding Strata to various IDEs and editors."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+def update_json_recursively(data: dict, keys: list, value) -> dict:
+    """Recursively update a nested dictionary, creating keys as needed.
+
+    Args:
+        data: The dictionary to update
+        keys: List of keys representing the path to update
+        value: The value to set at the path
+
+    Returns:
+        The updated dictionary
+    """
+    if not keys:
+        return data
+
+    if len(keys) == 1:
+        # Base case: set the value
+        key = keys[0]
+        if isinstance(data.get(key), dict) and isinstance(value, dict):
+            # Merge dictionaries if both are dict type
+            data[key] = {**data.get(key, {}), **value}
+        else:
+            data[key] = value
+        return data
+
+    # Recursive case: ensure intermediate keys exist
+    key = keys[0]
+    if key not in data:
+        data[key] = {}
+    elif not isinstance(data[key], dict):
+        # If the existing value is not a dict, replace it with dict
+        data[key] = {}
+
+    data[key] = update_json_recursively(data[key], keys[1:], value)
+    return data
+
+
+def ensure_json_config(config_path: Path) -> dict:
+    """Ensure JSON configuration file exists and return its content.
+
+    Args:
+        config_path: Path to the configuration file
+
+    Returns:
+        The configuration dictionary
+    """
+    # Create directory if it doesn't exist
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Load existing config or create empty one
+    if config_path.exists():
+        try:
+            with open(config_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError) as e:
+            print(
+                f"Warning: Could not read existing config {config_path}: {e}",
+                file=sys.stderr,
+            )
+            print("Creating new configuration file", file=sys.stderr)
+
+    return {}
+
+
+def save_json_config(config_path: Path, config: dict) -> None:
+    """Save JSON configuration to file.
+
+    Args:
+        config_path: Path to save the configuration
+        config: Configuration dictionary to save
+    """
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(config_path, "w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2, ensure_ascii=False)
+
+
+def check_cli_available(target: str) -> bool:
+    """Check if a CLI tool is available.
+
+    Args:
+        target: Name of the CLI tool to check
+
+    Returns:
+        True if the CLI tool is available, False otherwise
+    """
+    try:
+        if target == "vscode":
+            # Check for VSCode CLI (code command)
+            result = subprocess.run(
+                ["code", "--version"], capture_output=True, text=True, timeout=5
+            )
+        else:
+            result = subprocess.run(
+                [target, "--version"], capture_output=True, text=True, timeout=5
+            )
+
+        return result.returncode == 0
+    except (subprocess.SubprocessError, FileNotFoundError):
+        return False
+
+
+def add_strata_to_cursor(scope: str = "user") -> int:
+    """Add Strata to Cursor MCP configuration.
+
+    Args:
+        scope: Configuration scope (user, project, or local)
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Determine config path based on scope
+        if scope == "user":
+            # User scope: ~/.cursor/mcp.json
+            cursor_config_path = Path.home() / ".cursor" / "mcp.json"
+        elif scope in ["project", "local"]:
+            # Project scope: .cursor/mcp.json in current directory
+            cursor_config_path = Path.cwd() / ".cursor" / "mcp.json"
+        else:
+            print(
+                f"Error: Unsupported scope '{scope}' for Cursor. Supported: user, project, local",
+                file=sys.stderr,
+            )
+            return 1
+
+        print(
+            f"Adding Strata to Cursor with scope '{scope}' at {cursor_config_path}..."
+        )
+
+        # Load or create cursor configuration
+        cursor_config = ensure_json_config(cursor_config_path)
+
+        # Create Strata server configuration for Cursor
+        strata_server_config = {"command": "strata", "args": []}
+
+        # Update configuration using recursive update
+        cursor_config = update_json_recursively(
+            cursor_config, ["mcpServers", "strata"], strata_server_config
+        )
+
+        # Save updated configuration
+        save_json_config(cursor_config_path, cursor_config)
+        print("✓ Successfully added Strata to Cursor MCP configuration")
+        return 0
+
+    except (IOError, OSError) as e:
+        print(f"Error handling Cursor configuration: {e}", file=sys.stderr)
+        return 1
+
+
+def add_strata_to_vscode() -> int:
+    """Add Strata to VSCode MCP configuration.
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # VSCode uses JSON format: code --add-mcp '{"name":"strata","command":"strata"}'
+        mcp_config = {"name": "strata", "command": "strata"}
+        mcp_json = json.dumps(mcp_config)
+
+        print("Adding Strata to VSCode...")
+        cmd = ["code", "--add-mcp", mcp_json]
+        result = subprocess.run(cmd, capture_output=True, text=True)
+
+        if result.returncode == 0:
+            print("✓ Successfully added Strata to VSCode MCP configuration")
+            if result.stdout.strip():
+                print(result.stdout)
+            return 0
+        else:
+            print(
+                f"Error adding Strata to VSCode: {result.stderr.strip()}",
+                file=sys.stderr,
+            )
+            return result.returncode
+
+    except subprocess.SubprocessError as e:
+        print(f"Error running VSCode command: {e}", file=sys.stderr)
+        return 1
+
+
+def add_strata_to_claude_or_gemini(target: str, scope: str = "user") -> int:
+    """Add Strata to Claude or Gemini MCP configuration.
+
+    Args:
+        target: Target CLI tool (claude or gemini)
+        scope: Configuration scope
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Claude and Gemini use the original format
+        cmd = [target, "mcp", "add"]
+        cmd.extend(["--scope", scope])
+        cmd.extend(["strata", "strata"])
+
+        print(f"Adding Strata to {target} with scope '{scope}'...")
+        result = subprocess.run(cmd, capture_output=True, text=True)
+
+        if result.returncode == 0:
+            print(f"✓ Successfully added Strata to {target} MCP configuration")
+            if result.stdout.strip():
+                print(result.stdout)
+            return 0
+        else:
+            print(
+                f"Error adding Strata to {target}: {result.stderr.strip()}",
+                file=sys.stderr,
+            )
+            return result.returncode
+
+    except subprocess.SubprocessError as e:
+        print(f"Error running {target} command: {e}", file=sys.stderr)
+        return 1
+
+
+def add_strata_to_tool(target: str, scope: str = "user") -> int:
+    """Add Strata MCP server to specified tool configuration.
+
+    Args:
+        target: Target tool (claude, gemini, vscode, cursor)
+        scope: Configuration scope
+
+    Returns:
+        0 on success, 1 on error
+    """
+    target = target.lower()
+
+    # Validate target
+    if target not in ["claude", "gemini", "vscode", "cursor"]:
+        print(
+            f"Error: Unsupported target '{target}'. Supported targets: claude, gemini, vscode, cursor",
+            file=sys.stderr,
+        )
+        return 1
+
+    # VSCode doesn't support scope parameter
+    if target == "vscode" and scope != "user":
+        print(
+            "Warning: VSCode doesn't support scope parameter, using default behavior",
+            file=sys.stderr,
+        )
+
+    # Check if the target CLI is available (skip for cursor as we handle files directly)
+    if target != "cursor":
+        if not check_cli_available(target):
+            cli_name = "code" if target == "vscode" else target
+            print(
+                f"Error: {cli_name} CLI not found. Please install {cli_name} CLI first.",
+                file=sys.stderr,
+            )
+            return 1
+
+    # Handle each target
+    if target == "cursor":
+        return add_strata_to_cursor(scope)
+    elif target == "vscode":
+        return add_strata_to_vscode()
+    else:  # claude or gemini
+        return add_strata_to_claude_or_gemini(target, scope)