signalpilot-ai-internal 0.10.22__py3-none-any.whl → 0.11.24__py3-none-any.whl

signalpilot_ai_internal/handlers.py

@@ -8,6 +8,8 @@ from jupyter_server.base.handlers import APIHandler
 from jupyter_server.utils import url_path_join
 import tornado
 
+# Import controlled print function
+from .log_utils import print
 from .cache_service import get_cache_service
 from .cache_handlers import ChatHistoriesHandler, AppValuesHandler, CacheInfoHandler
 from .unified_database_schema_service import UnifiedDatabaseSchemaHandler, UnifiedDatabaseQueryHandler
@@ -28,6 +30,20 @@ from .mcp_handlers import (
     MCPToolEnableHandler,
     MCPConfigFileHandler
 )
+from .composio_handlers import (
+    IntegrationsHandler,
+    IntegrationConnectHandler,
+    IntegrationCompleteHandler,
+    IntegrationStatusHandler,
+    IntegrationDisconnectHandler,
+    IntegrationRefreshHandler,
+)
+from .database_config_handlers import (
+    DatabaseConfigsHandler,
+    DatabaseDefaultsHandler,
+    SignalPilotHomeInfoHandler,
+)
+from .signalpilot_home import get_user_rules_manager
 
 
 class HelloWorldHandler(APIHandler):
@@ -42,11 +58,205 @@ class HelloWorldHandler(APIHandler):
         }))
 
 
+class UserRulesHandler(APIHandler):
+    """Handler for user rules (snippets) stored as markdown files."""
+
+    @tornado.web.authenticated
+    def get(self, rule_id=None):
+        """Get all rules or a specific rule by ID."""
+        try:
+            rules_manager = get_user_rules_manager()
+
+            if not rules_manager.is_available():
+                self.set_status(503)
+                self.finish(json.dumps({
+                    "error": "User rules service not available",
+                    "message": "Rules directory is not accessible"
+                }))
+                return
+
+            if rule_id:
+                # Get specific rule
+                rule = rules_manager.get_rule(rule_id)
+                if rule is None:
+                    self.set_status(404)
+                    self.finish(json.dumps({
+                        "error": "Rule not found",
+                        "rule_id": rule_id
+                    }))
+                else:
+                    self.finish(json.dumps(rule))
+            else:
+                # Get all rules
+                rules = rules_manager.list_rules()
+                self.finish(json.dumps({
+                    "rules": rules,
+                    "count": len(rules)
+                }))
+
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": "Internal server error",
+                "message": str(e)
+            }))
+
+    @tornado.web.authenticated
+    def post(self, rule_id=None):
+        """Create a new rule or update an existing one."""
+        try:
+            rules_manager = get_user_rules_manager()
+
+            if not rules_manager.is_available():
+                self.set_status(503)
+                self.finish(json.dumps({
+                    "error": "User rules service not available",
+                    "message": "Rules directory is not accessible"
+                }))
+                return
+
+            # Parse request body
+            try:
+                body = json.loads(self.request.body.decode('utf-8'))
+            except json.JSONDecodeError:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "Invalid JSON in request body"
+                }))
+                return
+
+            if rule_id:
+                # Update existing rule
+                title = body.get('title')
+                content = body.get('content')
+                description = body.get('description')
+
+                result = rules_manager.update_rule(
+                    rule_id=rule_id,
+                    title=title,
+                    content=content,
+                    description=description
+                )
+
+                if result:
+                    self.finish(json.dumps({
+                        "success": True,
+                        "rule": result,
+                        "message": "Rule updated successfully"
+                    }))
+                else:
+                    self.set_status(404)
+                    self.finish(json.dumps({
+                        "error": "Rule not found",
+                        "rule_id": rule_id
+                    }))
+            else:
+                # Create new rule
+                title = body.get('title')
+                content = body.get('content', '')
+                description = body.get('description', '')
+                provided_id = body.get('id')
+
+                if not title:
+                    self.set_status(400)
+                    self.finish(json.dumps({
+                        "error": "Missing required field: title"
+                    }))
+                    return
+
+                result = rules_manager.create_rule(
+                    title=title,
+                    content=content,
+                    description=description,
+                    rule_id=provided_id
+                )
+
+                if result:
+                    self.set_status(201)
+                    self.finish(json.dumps({
+                        "success": True,
+                        "rule": result,
+                        "message": "Rule created successfully"
+                    }))
+                else:
+                    self.set_status(500)
+                    self.finish(json.dumps({
+                        "error": "Failed to create rule"
+                    }))
+
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": "Internal server error",
+                "message": str(e)
+            }))
+
+    @tornado.web.authenticated
+    def delete(self, rule_id=None):
+        """Delete a rule by ID."""
+        try:
+            rules_manager = get_user_rules_manager()
+
+            if not rules_manager.is_available():
+                self.set_status(503)
+                self.finish(json.dumps({
+                    "error": "User rules service not available",
+                    "message": "Rules directory is not accessible"
+                }))
+                return
+
+            if not rule_id:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "Rule ID is required for deletion"
+                }))
+                return
+
+            success = rules_manager.delete_rule(rule_id)
+
+            if success:
+                self.finish(json.dumps({
+                    "success": True,
+                    "message": f"Rule '{rule_id}' deleted successfully"
+                }))
+            else:
+                self.set_status(404)
+                self.finish(json.dumps({
+                    "error": "Rule not found",
+                    "rule_id": rule_id
+                }))
+
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": "Internal server error",
+                "message": str(e)
+            }))
+
+
+class UserRulesInfoHandler(APIHandler):
+    """Handler for user rules service information."""
+
+    @tornado.web.authenticated
+    def get(self):
+        """Get user rules service information."""
+        try:
+            rules_manager = get_user_rules_manager()
+            info = rules_manager.get_info()
+            self.finish(json.dumps(info))
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": "Internal server error",
+                "message": str(e)
+            }))
+
+
 class ReadAllFilesHandler(APIHandler):
     """Handler for reading all notebook and data files in the workspace"""
     
     # Common data file extensions
-    DATA_EXTENSIONS = {'.csv', '.json', '.xlsx', '.xls', '.parquet', '.pkl', '.pickle', 
+    DATA_EXTENSIONS = {'.csv', '.json', '.xlsx', '.xls', '.parquet',
                        '.feather', '.hdf5', '.h5', '.sql', '.db', '.sqlite', '.tsv', '.txt'}
     
     # Directories to exclude from search
@@ -192,9 +402,9 @@ class ReadAllFilesHandler(APIHandler):
         # Search for data file references
         # Common patterns: pd.read_csv('file.csv'), open('file.csv'), 'path/to/file.csv'
         patterns = [
-            r'["\']([^"\']+\.(?:csv|json|xlsx?|parquet|pkl|pickle|feather|hdf5|h5|sql|db|sqlite|tsv|txt))["\']',
-            r'read_(?:csv|json|excel|parquet|pickle|feather|hdf|sql|table)\(["\']([^"\']+)["\']',
-            r'to_(?:csv|json|excel|parquet|pickle|feather|hdf|sql)\(["\']([^"\']+)["\']',
+            r'["\']([^"\']+\.(?:csv|json|xlsx?|parquet|feather|hdf5|h5|sql|db|sqlite|tsv|txt))["\']',
+            r'read_(?:csv|json|excel|parquet|feather|hdf|sql|table)\(["\']([^"\']+)["\']',
+            r'to_(?:csv|json|excel|parquet|feather|hdf|sql)\(["\']([^"\']+)["\']',
         ]
         
         found_references = set()
@@ -451,18 +661,72 @@ class ScannedDirectoriesHandler(APIHandler):
 
 
 class WorkDirHandler(APIHandler):
-    """Handler for returning current working directory"""
+    """Handler for returning current working directory and setup manager type"""
 
     @tornado.web.authenticated
     def get(self):
         try:
-            self.finish(json.dumps({"workdir": os.getcwd()}))
+            # Detect the setup manager type
+            setup_manager = self._detect_setup_manager()
+
+            self.finish(json.dumps({
+                "workdir": os.getcwd(),
+                "setupManager": setup_manager
+            }))
         except Exception as e:
             self.set_status(500)
             self.finish(json.dumps({
                 "error": str(e)
             }))
 
+    def _detect_setup_manager(self) -> str:
+        """
+        Detect the Python environment manager being used.
+        Returns: 'conda', 'venv', 'uv', or 'system'
+        """
+        env_type, _ = self._detect_kernel_env()
+        return env_type
+
+    def _detect_kernel_env(self):
+        """
+        Detect the actual environment running this kernel,
+        not what shell env vars claim.
+        """
+        import sys
+
+        prefix = Path(sys.prefix)
+        executable = Path(sys.executable)
+
+        # Check pyvenv.cfg first — exists for venv/uv, not conda
+        pyvenv_cfg = prefix / 'pyvenv.cfg'
+        if pyvenv_cfg.exists():
+            try:
+                content = pyvenv_cfg.read_text().lower()
+                if 'uv' in content:
+                    return 'uv', str(prefix)
+                return 'venv', str(prefix)
+            except (IOError, OSError):
+                return 'venv', str(prefix)
+
+        # No pyvenv.cfg — check if we're in a conda env
+        # Conda envs live under <conda_root>/envs/<name>/ or are the base env
+        # Key marker: conda-meta directory exists
+        conda_meta = prefix / 'conda-meta'
+        if conda_meta.exists():
+            # Extract env name from path
+            if 'envs' in prefix.parts:
+                idx = prefix.parts.index('envs')
+                env_name = prefix.parts[idx + 1] if len(prefix.parts) > idx + 1 else 'base'
+            else:
+                env_name = 'base'
+            return 'conda', env_name
+
+        # Fallback: check if prefix differs from base (generic venv)
+        if sys.prefix != sys.base_prefix:
+            return 'venv', str(prefix)
+
+        return 'system', str(prefix)
+
 
 class TerminalExecuteHandler(APIHandler):
     """Handler for executing terminal commands"""
@@ -490,9 +754,15 @@ class TerminalExecuteHandler(APIHandler):
                 timeout=300
             )
 
-            def truncate_output(output: str, max_lines: int = 50) -> str:
+            def truncate_output(output: str, max_lines: int = 50, max_chars: int = 20000) -> str:
                 if not output:
                     return output
+                # First, truncate by character limit
+                if len(output) > max_chars:
+                    half = max_chars // 2
+                    truncated_chars = len(output) - max_chars
+                    output = output[:half] + f'\n... {truncated_chars} characters truncated ...\n' + output[-half:]
+                # Then, truncate by line count
                 lines = output.splitlines()
                 if len(lines) <= max_lines * 2:
                     return output
@@ -780,6 +1050,25 @@ def setup_handlers(web_app):
     mcp_server_disable_route = url_path_join(base_url, "signalpilot-ai-internal", "mcp", "servers", "([^/]+)", "disable")
     mcp_config_file_route = url_path_join(base_url, "signalpilot-ai-internal", "mcp", "config-file")
 
+    # Composio integration endpoints
+    integrations_route = url_path_join(base_url, "signalpilot-ai-internal", "integrations")
+    integration_connect_route = url_path_join(base_url, "signalpilot-ai-internal", "integrations", "([^/]+)", "connect")
+    integration_complete_route = url_path_join(base_url, "signalpilot-ai-internal", "integrations", "([^/]+)", "complete")
+    integration_status_route = url_path_join(base_url, "signalpilot-ai-internal", "integrations", "([^/]+)", "status")
+    integration_refresh_route = url_path_join(base_url, "signalpilot-ai-internal", "integrations", "([^/]+)", "refresh")
+    integration_disconnect_route = url_path_join(base_url, "signalpilot-ai-internal", "integrations", "([^/]+)")
+
+    # Database config endpoints (db.toml)
+    db_configs_route = url_path_join(base_url, "signalpilot-ai-internal", "db-configs")
+    db_configs_type_route = url_path_join(base_url, "signalpilot-ai-internal", "db-configs", "([^/]+)")
+    db_defaults_route = url_path_join(base_url, "signalpilot-ai-internal", "db-defaults")
+    signalpilot_home_info_route = url_path_join(base_url, "signalpilot-ai-internal", "home-info")
+
+    # User rules endpoints (markdown files in user-rules/)
+    user_rules_route = url_path_join(base_url, "signalpilot-ai-internal", "rules")
+    user_rule_route = url_path_join(base_url, "signalpilot-ai-internal", "rules", "([^/]+)")
+    user_rules_info_route = url_path_join(base_url, "signalpilot-ai-internal", "rules-info")
+
     handlers = [
         # Original endpoint
         (hello_route, HelloWorldHandler),
@@ -845,6 +1134,26 @@ def setup_handlers(web_app):
         (mcp_tool_enable_route, MCPToolEnableHandler),
         (mcp_server_enable_route, MCPServerEnableHandler),
         (mcp_server_disable_route, MCPServerDisableHandler),
+
+        # Composio integration endpoints
+        # Note: More specific routes should come before parameterized routes
+        (integrations_route, IntegrationsHandler),
+        (integration_connect_route, IntegrationConnectHandler),
+        (integration_complete_route, IntegrationCompleteHandler),
+        (integration_status_route, IntegrationStatusHandler),
+        (integration_refresh_route, IntegrationRefreshHandler),
+        (integration_disconnect_route, IntegrationDisconnectHandler),
+
+        # Database config endpoints (db.toml in cache_dir/connect/)
+        (db_configs_route, DatabaseConfigsHandler),
+        (db_configs_type_route, DatabaseConfigsHandler),
+        (db_defaults_route, DatabaseDefaultsHandler),
+        (signalpilot_home_info_route, SignalPilotHomeInfoHandler),
+
+        # User rules endpoints (markdown files in user-rules/)
+        (user_rules_route, UserRulesHandler),
+        (user_rule_route, UserRulesHandler),
+        (user_rules_info_route, UserRulesInfoHandler),
     ]
     
     web_app.add_handlers(host_pattern, handlers)
@@ -868,7 +1177,7 @@ def setup_handlers(web_app):
     # Register cleanup with web app
     import atexit
     atexit.register(cleanup_mcp_servers)
-    
+
     print("SignalPilot AI backend handlers registered:")
     print(f"  - Hello World: {hello_route}")
     print(f"  - Read All Files: {read_all_files_route}")

signalpilot_ai_internal/integrations_config.py

@@ -0,0 +1,256 @@
+"""
+Integration Configuration for Composio OAuth Apps
+This file contains static configuration for integrations with MCP servers.
+IMPORTANT: This file is NOT exposed to the frontend for security reasons.
+"""
+
+import base64
+import json
+import logging
+from typing import Dict, List, Any
+
+logger = logging.getLogger(__name__)
+
+# Integration configuration - defines how to set up MCP servers after OAuth
+INTEGRATION_CONFIG: Dict[str, Dict[str, Any]] = {
+    'notion': {
+        'id': 'notion',
+        'name': 'Notion',
+        'description': 'Connect to Notion workspaces for reading and searching pages',
+        'mcp_server_id': 'notion-integration',
+        'mcp_server_name': 'Notion (Composio)',
+        'mcp_command': 'npx',
+        'mcp_args': ['-y', '@notionhq/notion-mcp-server'],
+        # Maps Composio credential fields to MCP server env vars
+        'env_mapping': {
+            'access_token': 'NOTION_TOKEN',
+        },
+        'whitelisted_tools': [
+            'API-post-search',
+            'API-get-block-children',
+            'API-retrieve-a-page',
+            'API-retrieve-a-database',
+            'API-post-database-query',
+        ],
+    },
+    'slack': {
+        'id': 'slack',
+        'name': 'Slack',
+        'description': 'Connect to Slack workspaces for searching and reading messages',
+        'mcp_server_id': 'slack-integration',
+        'mcp_server_name': 'Slack (Composio)',
+        # Note: This requires the slack-mcp-server to be installed
+        # Using npx with a placeholder - user may need to adjust based on their setup
+        'mcp_command': 'npx',
+        'mcp_args': ['-y', 'slack-mcp-server@latest', '--transport', 'stdio'],
+        # Maps Composio credential fields to MCP server env vars
+        'env_mapping': {
+            'access_token': 'SLACK_MCP_XOXP_TOKEN',
+        },
+        'whitelisted_tools': [
+            'conversations_search_messages',
+            'conversations_history',
+            'conversations_replies',
+            'channels_list',
+        ],
+    },
+    'google': {
+        'id': 'google',
+        'name': 'Google Docs',
+        'description': 'Connect to Google Drive and Docs for searching and reading documents',
+        'mcp_server_id': 'google-integration',
+        'mcp_server_name': 'Google Docs (Composio)',
+        # Note: This requires uvx to be installed
+        # --single-user makes USER_GOOGLE_EMAIL the default, skipping email in tool calls
+        'mcp_command': 'uvx',
+        'mcp_args': ['workspace-mcp', '--tools', 'drive', 'docs', '--single-user'],
+        # Maps Composio credential fields to MCP server env vars
+        'env_mapping': {
+            'client_id': 'GOOGLE_OAUTH_CLIENT_ID',
+            'client_secret': 'GOOGLE_OAUTH_CLIENT_SECRET',
+            'access_token': 'GOOGLE_ACCESS_TOKEN',
+            'refresh_token': 'GOOGLE_REFRESH_TOKEN',
+            'email': 'USER_GOOGLE_EMAIL',  # User's Google email from OAuth
+        },
+        'whitelisted_tools': [
+            'start_google_auth',
+            'search_docs',
+            'get_doc_content',
+            'list_docs_in_folder',
+            'inspect_doc_structure',
+            'read_document_comments',
+            'create_document_comment',
+            'reply_to_document_comment',
+            'resolve_document_comment',
+            'search_drive_files',
+            'list_drive_items',
+            'get_drive_file_content',
+            'get_drive_file_download_url',
+            'list_drive_items_in_folder',
+        ],
+    },
+}
+
+
+def get_integration_config(integration_id: str) -> Dict[str, Any] | None:
+    """Get configuration for a specific integration."""
+    return INTEGRATION_CONFIG.get(integration_id)
+
+
+def get_all_integration_ids() -> List[str]:
+    """Get list of all integration IDs."""
+    return list(INTEGRATION_CONFIG.keys())
+
+
+def get_integration_info_for_frontend() -> List[Dict[str, str]]:
+    """
+    Get integration info safe to expose to frontend.
+    Only includes id, name, and description - no MCP commands or env mappings.
+    """
+    return [
+        {
+            'id': config['id'],
+            'name': config['name'],
+            'description': config['description'],
+        }
+        for config in INTEGRATION_CONFIG.values()
+    ]
+
+
+def get_mcp_server_config(integration_id: str, credentials: Dict[str, Any]) -> Dict[str, Any] | None:
+    """
+    Generate MCP server configuration for an integration.
+    Maps credentials from Composio to the appropriate env vars for the MCP server.
+
+    DEPRECATED: Use get_mcp_server_config_for_storage() and store tokens separately.
+    """
+    config = INTEGRATION_CONFIG.get(integration_id)
+    if not config:
+        return None
+
+    # Build environment variables from credentials
+    env = {}
+    for cred_field, env_var in config['env_mapping'].items():
+        if cred_field in credentials:
+            env[env_var] = credentials[cred_field]
+
+    return {
+        'id': config['mcp_server_id'],
+        'name': config['mcp_server_name'],
+        'command': config['mcp_command'],
+        'args': config['mcp_args'],
+        'env': env,
+        'enabled': True,
+        'enabledTools': config['whitelisted_tools'],
+    }
+
+
+def get_mcp_server_config_for_storage(integration_id: str) -> Dict[str, Any] | None:
+    """
+    Generate MCP server configuration for storage (WITHOUT credentials).
+    Tokens should be stored separately using OAuthTokenStore and injected at runtime.
+
+    Returns config suitable for mcp.json without sensitive data.
+    """
+    config = INTEGRATION_CONFIG.get(integration_id)
+    if not config:
+        return None
+
+    return {
+        'id': config['mcp_server_id'],
+        'name': config['mcp_server_name'],
+        'command': config['mcp_command'],
+        'args': config['mcp_args'],
+        # No env vars with tokens - they're stored securely elsewhere
+        'enabled': True,
+        'enabledTools': config['whitelisted_tools'],
+        # Mark as OAuth integration so MCP service knows to look up tokens
+        'isOAuthIntegration': True,
+    }
+
+
+def _extract_email_from_id_token(id_token: str) -> str | None:
+    """
+    Extract email from a Google OAuth id_token (JWT).
+
+    The id_token is a JWT with three parts: header.payload.signature
+    We decode the payload to get the email claim.
+    """
+    try:
+        # JWT has 3 parts separated by dots
+        parts = id_token.split('.')
+        if len(parts) != 3:
+            return None
+
+        # Decode the payload (second part)
+        # Add padding if needed for base64 decoding
+        payload_b64 = parts[1]
+        padding = 4 - len(payload_b64) % 4
+        if padding != 4:
+            payload_b64 += '=' * padding
+
+        payload_json = base64.urlsafe_b64decode(payload_b64)
+        payload = json.loads(payload_json)
+
+        # Extract email from claims
+        email = payload.get('email')
+        if email:
+            logger.debug(f"[Integrations] Extracted email from id_token: {email}")
+            return email
+
+        return None
+    except Exception as e:
+        logger.warning(f"[Integrations] Failed to extract email from id_token: {e}")
+        return None
+
+
+def build_env_from_credentials(integration_id: str, credentials: Dict[str, Any]) -> Dict[str, str]:
+    """
+    Build environment variables from Composio credentials.
+
+    Args:
+        integration_id: The integration ID
+        credentials: Credentials from Composio
+
+    Returns:
+        Dict of environment variable name -> value
+    """
+    config = INTEGRATION_CONFIG.get(integration_id)
+    if not config:
+        return {}
+
+    env = {}
+    for cred_field, env_var in config['env_mapping'].items():
+        if cred_field in credentials:
+            env[env_var] = credentials[cred_field]
+
+    # For Google integration, try to extract email from various sources if not directly available
+    if integration_id == 'google' and 'USER_GOOGLE_EMAIL' not in env:
+        email = None
+
+        # Try common field names for email
+        for field_name in ['email', 'userEmail', 'user_email', 'Email']:
+            if field_name in credentials:
+                email = credentials[field_name]
+                break
+
+        # Try to extract from id_token if not found directly
+        if not email:
+            id_token = credentials.get('id_token')
+            if id_token:
+                email = _extract_email_from_id_token(id_token)
+
+        if email:
+            env['USER_GOOGLE_EMAIL'] = email
+        else:
+            logger.warning(f"[Integrations] Could not find email in Google credentials. Keys: {list(credentials.keys())}")
+
+    return env
+
+
+def get_mcp_server_id_for_integration(integration_id: str) -> str | None:
+    """Get the MCP server ID for an integration."""
+    config = INTEGRATION_CONFIG.get(integration_id)
+    if config:
+        return config['mcp_server_id']
+    return None

signalpilot_ai_internal/log_utils.py

@@ -0,0 +1,31 @@
+"""
+Logging utilities for SignalPilot AI.
+Provides controlled print function that respects SIGNALPILOT_DEBUG environment variable.
+"""
+
+import builtins
+import os
+
+# Debug mode controlled by environment variable
+# Set SIGNALPILOT_DEBUG=1 to enable verbose logging (disabled by default)
+_DEBUG = os.environ.get('SIGNALPILOT_DEBUG', '0').lower() not in ('0', 'false', 'no')
+
+# Store the original print function
+_original_print = builtins.print
+
+
+def print(*args, **kwargs) -> None:
+    """Conditionally print messages based on debug mode.
+
+    Only prints when SIGNALPILOT_DEBUG environment variable is not set to 0/false/no.
+    Error messages (containing 'ERROR') are always printed regardless of debug mode.
+    """
+    message = ' '.join(str(arg) for arg in args)
+    # Always print errors, otherwise only print in debug mode
+    if 'ERROR' in message or _DEBUG:
+        _original_print(*args, **kwargs)
+
+
+def is_debug_enabled() -> bool:
+    """Check if debug mode is enabled."""
+    return _DEBUG