signalpilot-ai-internal 0.10.22__py3-none-any.whl → 0.11.24__py3-none-any.whl

signalpilot_ai_internal/mcp_handlers.py

@@ -8,6 +8,7 @@ import traceback
 import tornado
 from jupyter_server.base.handlers import APIHandler
 from .mcp_service import get_mcp_service
+from .oauth_token_store import get_oauth_token_store
 
 logger = logging.getLogger(__name__)
 
@@ -23,8 +24,9 @@ class MCPServersHandler(APIHandler):
         """Get all configured MCP servers"""
         try:
             mcp_service = get_mcp_service()
+            token_store = get_oauth_token_store()
             configs = mcp_service.load_all_configs()
-            
+
             # Add connection status to each server
             servers = []
             for server_id, config in configs.items():
@@ -33,13 +35,24 @@ class MCPServersHandler(APIHandler):
                     'status': mcp_service.get_connection_status(server_id),
                     'enabled': config.get('enabled', True)
                 }
-                
+
                 # Add tool count if connected
                 if server_id in mcp_service.tools_cache:
                     server_info['toolCount'] = len(mcp_service.tools_cache[server_id])
-                
+
+                # Check if this is an OAuth integration and add the integration ID
+                is_oauth = config.get('isOAuthIntegration', False)
+                if not is_oauth:
+                    is_oauth = token_store.is_oauth_server(server_id)
+
+                if is_oauth:
+                    server_info['isOAuthIntegration'] = True
+                    integration_id = token_store.get_integration_id(server_id)
+                    if integration_id:
+                        server_info['integrationId'] = integration_id
+
                 servers.append(server_info)
-            
+
             self.finish(json.dumps({
                 'servers': servers
             }))
@@ -326,27 +339,38 @@ class MCPAllToolsHandler(APIHandler):
 
 class MCPToolCallHandler(APIHandler):
     """Handler for calling MCP tools"""
-    
+
     @tornado.web.authenticated
     async def post(self):
         """Call a tool on an MCP server"""
         try:
             data = json.loads(self.request.body.decode('utf-8'))
-            
+
             server_id = data.get('server_id')
             tool_name = data.get('tool_name')
             arguments = data.get('arguments', {})
-            
+
             if not server_id or not tool_name:
                 self.set_status(400)
                 self.finish(json.dumps({
                     'error': 'server_id and tool_name are required'
                 }))
                 return
-            
+
+            # Workaround: Inject user_google_email for Google tools
+            # The MCP server requires this parameter even with --single-user mode
+            # See: https://github.com/taylorwilsdon/google_workspace_mcp/issues/338
+            token_store = get_oauth_token_store()
+            if token_store.is_oauth_server(server_id):
+                integration_id = token_store.get_integration_id(server_id)
+                if integration_id == 'google' and 'user_google_email' not in arguments:
+                    oauth_env = token_store.get_tokens(server_id)
+                    if oauth_env and 'USER_GOOGLE_EMAIL' in oauth_env:
+                        arguments['user_google_email'] = oauth_env['USER_GOOGLE_EMAIL']
+
             mcp_service = get_mcp_service()
             result = await mcp_service.call_tool(server_id, tool_name, arguments)
-            
+
             self.finish(json.dumps({
                 'success': True,
                 'result': result

signalpilot_ai_internal/mcp_service.py

@@ -19,7 +19,8 @@ import os
 from typing import Dict, List, Optional, Any
 from pathlib import Path
 import aiohttp
-from .cache_service import get_cache_service, RobustFileOperations
+from .signalpilot_home import get_signalpilot_home
+from .oauth_token_store import get_oauth_token_store
 
 logger = logging.getLogger(__name__)
 
@@ -96,10 +97,7 @@ class MCPConnectionService:
     def __init__(self):
         self.connections: Dict[str, 'MCPConnection'] = {}
         self.tools_cache: Dict[str, List[Dict]] = {}
-        self.cache = get_cache_service()
-        self.mcp_config_key = 'mcp_servers'
-        self._migrated_from_cache = False
-        self._ensure_migration()
+        self.home_manager = get_signalpilot_home()
     
     @classmethod
     def get_instance(cls):
@@ -108,22 +106,6 @@ class MCPConnectionService:
             cls._instance = MCPConnectionService()
         return cls._instance
     
-    def get_mcp_config_path(self) -> Path:
-        """Get the path to the MCP configuration JSON file (Cursor format)"""
-        # Use the same cache directory as other SignalPilot files for consistency
-        from .cache_service import CacheDirectoryManager
-        cache_dir = CacheDirectoryManager.find_usable_cache_directory()
-        if cache_dir:
-            return cache_dir / 'mcp.json'
-        else:
-            # Fallback to old location if cache directory not available
-            if platform.system() == 'Windows':
-                config_dir = Path(os.environ.get('USERPROFILE', Path.home())) / '.signalpilot-ai-internal'
-            else:
-                config_dir = Path.home() / '.signalpilot-ai-internal'
-            config_dir.mkdir(parents=True, exist_ok=True)
-            return config_dir / 'mcp.json'
-    
     def _infer_server_type(self, config: Dict) -> str:
         """Infer server type from config structure (Cursor format)"""
         if 'command' in config:
@@ -164,16 +146,17 @@ class MCPConnectionService:
         """Convert internal format to Cursor schema format"""
         # Remove internal-only fields
         storage_config = {}
-        
+
         # Copy relevant fields based on type
         server_type = config.get('type', 'command')
-        
+
         if server_type == 'command':
             if 'command' in config:
                 storage_config['command'] = config['command']
             if 'args' in config:
                 storage_config['args'] = config['args']
-            if 'env' in config:
+            # Only store env if NOT an OAuth integration (OAuth tokens are stored securely elsewhere)
+            if 'env' in config and not config.get('isOAuthIntegration', False):
                 storage_config['env'] = config['env']
         else:  # http/sse
             if 'name' in config:
@@ -182,86 +165,37 @@ class MCPConnectionService:
                 storage_config['url'] = config['url']
             if 'token' in config:
                 storage_config['token'] = config['token']
-        
+
         # Add enabled if not default (True)
         enabled = config.get('enabled', True)
         if not enabled:
             storage_config['enabled'] = False
-        
+
         # Add enabledTools if present
         if 'enabledTools' in config:
             storage_config['enabledTools'] = config['enabledTools']
-        
+
+        # Mark as OAuth integration if set (tokens stored securely elsewhere)
+        if config.get('isOAuthIntegration', False):
+            storage_config['isOAuthIntegration'] = True
+
         return storage_config
     
-    def _ensure_migration(self):
-        """Migrate from cache-based storage to JSON file if needed"""
-        if self._migrated_from_cache:
-            return
-        
-        config_path = self.get_mcp_config_path()
-        
-        # If JSON file already exists, migration already done
-        if config_path.exists():
-            self._migrated_from_cache = True
-            return
-        
-        # Try to migrate from cache
-        try:
-            cache_data = self.cache.get_app_value(self.mcp_config_key)
-            if cache_data:
-                old_configs = json.loads(cache_data)
-                if old_configs:
-                    logger.info(f"[MCP] Migrating {len(old_configs)} servers from cache to JSON file")
-                    
-                    # Convert to Cursor format
-                    cursor_format = {'mcpServers': {}}
-                    for server_id, config in old_configs.items():
-                        # Normalize old config
-                        if 'id' not in config:
-                            config['id'] = server_id
-                        if 'enabled' not in config:
-                            config['enabled'] = True
-                        
-                        # Convert to storage format
-                        storage_config = self._normalize_config_for_storage(config)
-                        cursor_format['mcpServers'][server_id] = storage_config
-                    
-                    # Write to JSON file
-                    if RobustFileOperations.safe_write_json(config_path, cursor_format):
-                        logger.info(f"[MCP] Successfully migrated to {config_path}")
-                        # Optionally delete old cache entry (commented for safety)
-                        # self.cache.delete_app_value(self.mcp_config_key)
-                    else:
-                        logger.error(f"[MCP] Failed to write migrated config to {config_path}")
-        except Exception as e:
-            logger.warning(f"[MCP] Migration from cache failed: {e}, starting fresh")
-        
-        self._migrated_from_cache = True
-    
     def save_server_config(self, server_config: Dict) -> Dict:
         """Save MCP server configuration to JSON file (Cursor format)"""
         try:
             # Ensure server has an ID
             if 'id' not in server_config:
                 server_config['id'] = str(uuid.uuid4())
-            
+
             server_id = server_config['id']
-            
-            # Load existing configs
-            config_path = self.get_mcp_config_path()
-            cursor_data = RobustFileOperations.safe_read_json(config_path, {})
-            mcp_servers = cursor_data.get('mcpServers', {})
-            
-            # Convert to storage format and add/update
+
+            # Convert to storage format and save
             storage_config = self._normalize_config_for_storage(server_config)
-            mcp_servers[server_id] = storage_config
-            
-            # Write back to file
-            cursor_data['mcpServers'] = mcp_servers
-            if not RobustFileOperations.safe_write_json(config_path, cursor_data):
-                raise RuntimeError(f"Failed to write MCP config to {config_path}")
-            
+
+            if not self.home_manager.set_mcp_server(server_id, storage_config):
+                raise RuntimeError(f"Failed to write MCP config")
+
             logger.info(f"Saved MCP server config: {server_config.get('name', server_id)}")
             return server_config
         except Exception as e:
@@ -272,22 +206,14 @@ class MCPConnectionService:
     def load_all_configs(self) -> Dict[str, Dict]:
         """Load all MCP server configurations from JSON file (Cursor format)"""
         try:
-            config_path = self.get_mcp_config_path()
-            
-            if not config_path.exists():
-                return {}
-            
-            # Read JSON file
-            cursor_data = RobustFileOperations.safe_read_json(config_path, {})
-            
-            # Extract mcpServers object
-            mcp_servers = cursor_data.get('mcpServers', {})
-            
+            # Get all MCP servers from home manager
+            mcp_servers = self.home_manager.get_mcp_servers()
+
             # Convert from Cursor format to internal format
             configs = {}
             for server_id, server_config in mcp_servers.items():
                 configs[server_id] = self._normalize_config_from_storage(server_id, server_config)
-            
+
             return configs
         except Exception as e:
             logger.error(f"Error loading MCP configs: {e}")
@@ -302,27 +228,12 @@ class MCPConnectionService:
     def delete_server_config(self, server_id: str) -> bool:
         """Delete a server configuration from JSON file"""
         try:
-            config_path = self.get_mcp_config_path()
-            
-            if not config_path.exists():
-                return False
-            
-            # Load current config
-            cursor_data = RobustFileOperations.safe_read_json(config_path, {})
-            mcp_servers = cursor_data.get('mcpServers', {})
-            
-            if server_id in mcp_servers:
-                del mcp_servers[server_id]
-                
-                # Write back to file
-                cursor_data['mcpServers'] = mcp_servers
-                if not RobustFileOperations.safe_write_json(config_path, cursor_data):
-                    raise RuntimeError(f"Failed to write MCP config to {config_path}")
-                
+            # Remove from home manager
+            if self.home_manager.remove_mcp_server(server_id):
                 # Also disconnect if connected
                 if server_id in self.connections:
                     asyncio.create_task(self.disconnect(server_id))
-                
+
                 logger.info(f"Deleted MCP server config: {server_id}")
                 return True
             return False
@@ -502,7 +413,15 @@ class MCPConnectionService:
         """Get server information for response"""
         tools = self.tools_cache.get(server_id, [])
         enabled_tools = config.get('enabledTools', [])
-        return {
+
+        # Check if this is an OAuth integration
+        is_oauth = config.get('isOAuthIntegration', False)
+        if not is_oauth:
+            # Also check token store in case the flag wasn't set
+            token_store = get_oauth_token_store()
+            is_oauth = token_store.is_oauth_server(server_id)
+
+        result = {
             'serverId': server_id,
             'name': config.get('name', server_id),
             'status': self.get_connection_status(server_id),
@@ -512,6 +431,14 @@ class MCPConnectionService:
             'enabled': config.get('enabled', True),
             'enabledTools': enabled_tools
         }
+
+        # Add OAuth info if it's an OAuth integration
+        if is_oauth:
+            result['isOAuthIntegration'] = True
+            token_store = get_oauth_token_store()
+            result['integrationId'] = token_store.get_integration_id(server_id)
+
+        return result
     
     def enable_server(self, server_id: str) -> bool:
         """Enable an MCP server"""
@@ -678,67 +605,65 @@ class MCPConnectionService:
     def update_config_file(self, new_json_content: str) -> Dict[str, Any]:
         """Update the entire config file and apply diff-based changes"""
         try:
-            config_path = self.get_mcp_config_path()
-            
             # Parse new JSON
             try:
                 new_data = json.loads(new_json_content)
             except json.JSONDecodeError as e:
                 raise ValueError(f"Invalid JSON: {e}")
-            
+
             if 'mcpServers' not in new_data:
                 raise ValueError("JSON must contain 'mcpServers' object")
-            
+
             new_servers = new_data.get('mcpServers', {})
-            
+
             # Load current configs
             old_configs = self.load_all_configs()
             old_server_ids = set(old_configs.keys())
             new_server_ids = set(new_servers.keys())
-            
+
             changes = {
                 'added': [],
                 'removed': [],
                 'modified': [],
                 'enabled_changes': []
             }
-            
+
             # Detect added servers
             for server_id in new_server_ids - old_server_ids:
                 changes['added'].append(server_id)
-            
+
             # Detect removed servers
             for server_id in old_server_ids - new_server_ids:
                 changes['removed'].append(server_id)
                 # Disconnect removed servers
                 if server_id in self.connections:
                     asyncio.create_task(self.disconnect(server_id))
-            
+
             # Detect modified servers
             for server_id in new_server_ids & old_server_ids:
                 old_config = old_configs[server_id]
                 new_storage_config = new_servers[server_id]
                 new_config = self._normalize_config_from_storage(server_id, new_storage_config)
-                
+
                 # Check if enabled status changed
                 old_enabled = old_config.get('enabled', True)
                 new_enabled = new_config.get('enabled', True)
-                
+
                 if old_enabled != new_enabled:
                     changes['enabled_changes'].append({
                         'server_id': server_id,
                         'old_enabled': old_enabled,
                         'new_enabled': new_enabled
                     })
-                
+
                 # Check if config changed (simple comparison)
                 old_storage = self._normalize_config_for_storage(old_config)
                 if old_storage != new_storage_config:
                     changes['modified'].append(server_id)
-            
-            # Write new config to file
-            if not RobustFileOperations.safe_write_json(config_path, new_data):
-                raise RuntimeError(f"Failed to write updated config to {config_path}")
+
+            # Write new config to file using home manager
+            if not self.home_manager.write_mcp_config(new_data):
+                raise RuntimeError(f"Failed to write updated config")
             
             # Apply changes asynchronously
             async def apply_changes():
@@ -793,11 +718,7 @@ class MCPConnectionService:
     def get_config_file_content(self) -> str:
         """Get the raw JSON file content"""
         try:
-            config_path = self.get_mcp_config_path()
-            if not config_path.exists():
-                return json.dumps({'mcpServers': {}}, indent=2)
-            
-            cursor_data = RobustFileOperations.safe_read_json(config_path, {'mcpServers': {}})
+            cursor_data = self.home_manager.read_mcp_config()
             return json.dumps(cursor_data, indent=2)
         except Exception as e:
             logger.error(f"Error reading config file: {e}")
@@ -850,10 +771,26 @@ class MCPCommandConnection(MCPConnection):
             command = self.config.get('command')
             args = self.config.get('args', [])
             env = self.config.get('env', {})
-            
+
             if not command:
                 raise ValueError("Command is required for command-based MCP")
-            
+
+            # Check if this is an OAuth integration and inject tokens from secure store
+            is_oauth = self.config.get('isOAuthIntegration', False)
+            token_store = get_oauth_token_store()
+
+            # Also check token store directly in case flag is not set
+            if not is_oauth:
+                is_oauth = token_store.is_oauth_server(self.server_id)
+
+            if is_oauth:
+                oauth_env = token_store.get_tokens(self.server_id)
+                if oauth_env:
+                    logger.debug(f"[MCP] Injecting OAuth tokens for server {self.server_id}")
+                    env = {**env, **oauth_env}  # OAuth tokens override any existing env vars
+                else:
+                    logger.warning(f"[MCP] Server {self.server_id} is marked as OAuth but no tokens found in store")
+
             # Merge environment variables
             import os
             import shlex
@@ -1071,7 +1008,22 @@ class MCPCommandConnection(MCPConnection):
                 logger.error(f"[MCP] Error during disconnect: {e}")
             finally:
                 self.connected = False
-    
+
+    def is_connected(self) -> bool:
+        """Check if connected - also verify subprocess is still alive"""
+        if not self.connected:
+            return False
+        # Check if subprocess is still running
+        if self.process is None:
+            return False
+        poll_result = self.process.poll()
+        if poll_result is not None:
+            # Process has exited, update connected flag
+            logger.warning(f"[MCP] Process for {self.server_id} has exited with code {poll_result}")
+            self.connected = False
+            return False
+        return True
+
     async def list_tools(self) -> List[Dict]:
         """List tools via JSON-RPC"""
         try:

signalpilot_ai_internal/oauth_token_store.py

@@ -0,0 +1,141 @@
+"""
+OAuth Token Store - Secure storage for OAuth tokens in .env format
+Stores tokens at <cache_dir>/connect/.env
+(e.g., ~/Library/Caches/SignalPilotAI/connect/.env on macOS)
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+from .signalpilot_home import get_signalpilot_home
+
+logger = logging.getLogger(__name__)
+
+
+class OAuthTokenStore:
+    """
+    Secure storage for OAuth tokens using .env format.
+    Tokens are stored with server-id prefixes for namespacing.
+    """
+
+    _instance = None
+
+    def __init__(self):
+        self._home_manager = get_signalpilot_home()
+
+    @classmethod
+    def get_instance(cls) -> 'OAuthTokenStore':
+        """Get singleton instance."""
+        if cls._instance is None:
+            cls._instance = OAuthTokenStore()
+        return cls._instance
+
+    def store_tokens(self, integration_id: str, mcp_server_id: str, env_vars: Dict[str, str]):
+        """
+        Store OAuth tokens for an integration.
+
+        Args:
+            integration_id: The integration ID (e.g., 'notion', 'slack')
+            mcp_server_id: The MCP server ID associated with this integration
+            env_vars: Environment variables containing tokens
+        """
+        # Store the registry entry (mapping server_id -> integration_id)
+        self._home_manager.set_oauth_registry_entry(mcp_server_id, integration_id)
+
+        # Store the actual tokens
+        self._home_manager.set_oauth_tokens(mcp_server_id, env_vars)
+
+        logger.info(f"[OAuthTokenStore] Stored tokens for {mcp_server_id}")
+
+    def get_tokens(self, mcp_server_id: str) -> Optional[Dict[str, str]]:
+        """
+        Get OAuth tokens for an MCP server.
+
+        Args:
+            mcp_server_id: The MCP server ID
+
+        Returns:
+            Environment variables dict or None if not found
+        """
+        return self._home_manager.get_oauth_tokens(mcp_server_id)
+
+    def get_integration_id(self, mcp_server_id: str) -> Optional[str]:
+        """
+        Get the integration ID for an MCP server.
+
+        Args:
+            mcp_server_id: The MCP server ID
+
+        Returns:
+            Integration ID or None if not found
+        """
+        registry = self._home_manager.get_oauth_registry()
+        return registry.get(mcp_server_id)
+
+    def is_oauth_server(self, mcp_server_id: str) -> bool:
+        """
+        Check if an MCP server is an OAuth integration.
+
+        Args:
+            mcp_server_id: The MCP server ID
+
+        Returns:
+            True if this server has stored OAuth tokens
+        """
+        registry = self._home_manager.get_oauth_registry()
+        return mcp_server_id in registry
+
+    def remove_tokens(self, mcp_server_id: str) -> bool:
+        """
+        Remove OAuth tokens for an MCP server.
+
+        Args:
+            mcp_server_id: The MCP server ID
+
+        Returns:
+            True if tokens were removed, False if not found
+        """
+        # Remove from registry
+        self._home_manager.remove_oauth_registry_entry(mcp_server_id)
+
+        # Remove the tokens
+        result = self._home_manager.remove_oauth_tokens(mcp_server_id)
+
+        if result:
+            logger.info(f"[OAuthTokenStore] Removed tokens for {mcp_server_id}")
+
+        return result
+
+    def update_tokens(self, mcp_server_id: str, env_vars: Dict[str, str]) -> bool:
+        """
+        Update OAuth tokens for an existing MCP server.
+
+        Args:
+            mcp_server_id: The MCP server ID
+            env_vars: New environment variables containing tokens
+
+        Returns:
+            True if tokens were updated, False if server not found
+        """
+        if not self.is_oauth_server(mcp_server_id):
+            logger.warning(f"[OAuthTokenStore] Server {mcp_server_id} not found for update")
+            return False
+
+        result = self._home_manager.set_oauth_tokens(mcp_server_id, env_vars)
+        if result:
+            logger.info(f"[OAuthTokenStore] Updated tokens for {mcp_server_id}")
+        return result
+
+    def get_all_oauth_servers(self) -> Dict[str, str]:
+        """
+        Get mapping of all OAuth MCP server IDs to their integration IDs.
+
+        Returns:
+            Dict mapping mcp_server_id -> integration_id
+        """
+        return self._home_manager.get_oauth_registry()
+
+
+def get_oauth_token_store() -> OAuthTokenStore:
+    """Get the singleton instance of the OAuth token store."""
+    return OAuthTokenStore.get_instance()

signalpilot_ai_internal/schema_search_config.yml

@@ -1,32 +1,38 @@
 logging:
-  level: 'WARNING'
+  level: "WARNING"
 
 embedding:
-  location: 'memory'
-  model: 'multi-qa-MiniLM-L6-cos-v1'
-  metric: 'cosine'
+  location: "memory" # Options: "memory", "vectordb" (coming soon)
+  model: "multi-qa-MiniLM-L6-cos-v1"
+  metric: "cosine" # Options: "cosine", "euclidean", "manhattan", "dot"
   batch_size: 32
   show_progress: false
-  cache_dir: '/tmp/.schema_search_cache'
+  cache_dir: "/tmp/.schema_search_cache"
 
 chunking:
-  strategy: 'raw'
+  strategy: "raw" # Options: "raw", "llm"
   max_tokens: 256
   overlap_tokens: 50
-  model: 'gpt-4o-mini'
+  model: "gpt-4o-mini"
 
 search:
-  strategy: 'bm25'
+  # Search strategy: "semantic" (embeddings), "bm25" (BM25 lexical), "fuzzy" (fuzzy string matching), "hybrid" (semantic + bm25)
+  strategy: "bm25"
   initial_top_k: 20
   rerank_top_k: 5
-  semantic_weight: 0.67
-  hops: 1
+  semantic_weight: 0.67 # For hybrid search (bm25_weight = 1 - semantic_weight)
+  hops: 1 # Number of foreign key hops for graph expansion (0-2 recommended)
 
 reranker:
-  model: null
+  # CrossEncoder model for reranking. Set to null to disable reranking
+  model: null # "Alibaba-NLP/gte-reranker-modernbert-base"
 
 schema:
   include_columns: true
   include_indices: true
   include_foreign_keys: true
   include_constraints: true
+
+output:
+  format: "markdown" # Options: "json", "markdown"
+  limit: 5 # Default number of results to return