pylance-mcp-server 1.0.0

package/mcp_server/cloud_sync.py

@@ -0,0 +1,427 @@
+# ============================================================
+# CLOUD TRAINING DATA SYNC (DISABLED FOR INITIAL LAUNCH)
+# ============================================================
+# This module syncs anonymized training data from local SQLite 
+# to cloud PostgreSQL for aggregate analysis across all customers.
+# 
+# STATUS: Not implemented - disabled for initial product launch
+# REASON: Focus on core MCP functionality first, add later
+# 
+# To enable: Uncomment endpoints in api-wrapper/index.js and
+# uncomment training tables in api-wrapper/schema.sql
+# ============================================================
+
+"""
+Cloud Sync for Training Data
+
+Syncs anonymized training data from local SQLite to Azure PostgreSQL/Supabase.
+Users have full control over what data is shared.
+"""
+
+import json
+import re
+import hashlib
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from datetime import datetime
+import requests
+
+
+class DataAnonymizer:
+    """Anonymize sensitive data before cloud sync."""
+    
+    def __init__(self):
+        self.path_salt = hashlib.sha256(str(datetime.now()).encode()).hexdigest()[:8]
+    
+    def anonymize_file_path(self, file_path: str) -> str:
+        """
+        Anonymize file paths while preserving structure.
+        
+        /home/john/projects/api_server/src/models.py
+        → /workspace/src/models.py
+        """
+        if not file_path:
+            return file_path
+        
+        path = Path(file_path)
+        
+        # Remove username/home directory
+        parts = path.parts
+        
+        # Find common project structure markers
+        markers = ['src', 'lib', 'app', 'tests', 'api', 'models', 'views', 'controllers']
+        
+        for i, part in enumerate(parts):
+            if part.lower() in markers:
+                # Keep from this marker onwards
+                return '/workspace/' + '/'.join(parts[i:])
+        
+        # If no marker found, just keep filename
+        return '/workspace/' + path.name
+    
+    def anonymize_workspace_root(self, workspace_root: str) -> str:
+        """
+        Anonymize workspace root path.
+        
+        /home/john/projects/my_company_api
+        → /workspace/project_abc123
+        """
+        if not workspace_root:
+            return workspace_root
+        
+        # Create consistent hash of path
+        path_hash = hashlib.md5(workspace_root.encode()).hexdigest()[:8]
+        return f"/workspace/project_{path_hash}"
+    
+    def anonymize_code_content(self, content: str) -> str:
+        """
+        Anonymize code content - remove sensitive strings, keep structure.
+        
+        Returns: Anonymized version with placeholders
+        """
+        if not content or len(content) > 5000:
+            # Don't sync large code blocks
+            return "[CODE_CONTENT_REDACTED]"
+        
+        # Remove common sensitive patterns
+        patterns = [
+            (r'(api[_-]?key|password|secret|token)\s*=\s*["\']([^"\']+)["\']', r'\1="[REDACTED]"'),
+            (r'https?://[^\s]+', '[URL_REDACTED]'),
+            (r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', '[IP_REDACTED]'),
+            (r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '[EMAIL_REDACTED]'),
+        ]
+        
+        anonymized = content
+        for pattern, replacement in patterns:
+            anonymized = re.sub(pattern, replacement, anonymized, flags=re.IGNORECASE)
+        
+        return anonymized
+    
+    def anonymize_prompt(self, prompt: str) -> str:
+        """
+        Anonymize user prompts - remove file paths and sensitive info, keep intent.
+        
+        "Add type hints to /home/john/api/routes.py"
+        → "Add type hints to routes.py"
+        """
+        if not prompt:
+            return prompt
+        
+        # Remove absolute paths, keep filenames
+        anonymized = re.sub(r'[/\\][^\s]+[/\\]([^/\\]+\.py)', r'\1', prompt)
+        
+        # Remove URLs
+        anonymized = re.sub(r'https?://[^\s]+', '[URL]', anonymized)
+        
+        # Remove emails
+        anonymized = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '[EMAIL]', anonymized)
+        
+        # Remove potential secrets
+        anonymized = re.sub(r'(token|key|password|secret)[:\s]*["\']?[a-zA-Z0-9_-]{16,}["\']?', r'\1=[REDACTED]', anonymized, flags=re.IGNORECASE)
+        
+        return anonymized
+    
+    def anonymize_conversation(self, conversation: Dict[str, Any]) -> Dict[str, Any]:
+        """Anonymize entire conversation record."""
+        anonymized = conversation.copy()
+        
+        # Anonymize paths
+        if 'workspace_root' in anonymized:
+            anonymized['workspace_root'] = self.anonymize_workspace_root(anonymized['workspace_root'])
+        
+        if 'venv_path' in anonymized:
+            anonymized['venv_path'] = self.anonymize_file_path(anonymized['venv_path'])
+        
+        # Remove session ID (too identifying)
+        if 'session_id' in anonymized:
+            anonymized['session_id'] = hashlib.md5(anonymized['session_id'].encode()).hexdigest()[:16]
+        
+        return anonymized
+    
+    def anonymize_message(self, message: Dict[str, Any]) -> Dict[str, Any]:
+        """Anonymize message record."""
+        anonymized = message.copy()
+        
+        # Anonymize content based on role
+        if 'content' in anonymized:
+            if anonymized.get('role') == 'user':
+                anonymized['content'] = self.anonymize_prompt(anonymized['content'])
+            else:
+                # For assistant, keep structure but remove specifics
+                content = anonymized['content']
+                if len(content) > 500:
+                    # Summarize long responses
+                    anonymized['content'] = content[:500] + '...[TRUNCATED]'
+        
+        return anonymized
+    
+    def anonymize_tool_call(self, tool_call: Dict[str, Any]) -> Dict[str, Any]:
+        """Anonymize tool call record."""
+        anonymized = tool_call.copy()
+        
+        # Parse and anonymize arguments
+        if 'arguments' in anonymized and anonymized['arguments']:
+            try:
+                args = json.loads(anonymized['arguments'])
+                
+                # Anonymize file paths in arguments
+                if 'file_path' in args:
+                    args['file_path'] = self.anonymize_file_path(args['file_path'])
+                
+                # Anonymize content
+                if 'content' in args:
+                    args['content'] = '[CONTENT_REDACTED]'
+                
+                # Keep other args like line numbers, character positions
+                anonymized['arguments'] = json.dumps(args)
+            except json.JSONDecodeError:
+                anonymized['arguments'] = '[INVALID_JSON]'
+        
+        # Anonymize result (keep structure, remove content)
+        if 'result' in anonymized and anonymized['result']:
+            try:
+                result = json.loads(anonymized['result'])
+                if isinstance(result, dict):
+                    # Keep keys, redact values
+                    result = {k: '[REDACTED]' if isinstance(v, str) and len(v) > 50 else v 
+                             for k, v in result.items()}
+                elif isinstance(result, list):
+                    result = f"[{len(result)} items]"
+                anonymized['result'] = json.dumps(result)
+            except json.JSONDecodeError:
+                anonymized['result'] = '[INVALID_JSON]'
+        
+        return anonymized
+    
+    def anonymize_file_modification(self, file_mod: Dict[str, Any]) -> Dict[str, Any]:
+        """Anonymize file modification record."""
+        anonymized = file_mod.copy()
+        
+        if 'file_path' in anonymized:
+            anonymized['file_path'] = self.anonymize_file_path(anonymized['file_path'])
+        
+        return anonymized
+
+
+class CloudSync:
+    """Sync training data to cloud PostgreSQL database."""
+    
+    def __init__(self, api_url: str, api_key: Optional[str] = None):
+        """
+        Initialize cloud sync.
+        
+        Args:
+            api_url: URL of the API endpoint (e.g., https://api.pylancemcp.com)
+            api_key: Optional API key for authentication
+        """
+        self.api_url = api_url.rstrip('/')
+        self.api_key = api_key
+        self.anonymizer = DataAnonymizer()
+    
+    def preview_anonymization(self, local_logger, conversation_id: int) -> Dict[str, Any]:
+        """
+        Preview what data will be sent before syncing.
+        
+        Args:
+            local_logger: ConversationLogger instance
+            conversation_id: ID of conversation to preview
+        
+        Returns:
+            Dictionary showing original vs anonymized data
+        """
+        conversation = local_logger.get_conversation(conversation_id)
+        messages = local_logger.get_messages(conversation_id)
+        tool_calls = local_logger.get_tool_calls(conversation_id)
+        
+        preview = {
+            'conversation': {
+                'original': conversation,
+                'anonymized': self.anonymizer.anonymize_conversation(conversation)
+            },
+            'messages': {
+                'count': len(messages),
+                'sample_original': messages[0] if messages else None,
+                'sample_anonymized': self.anonymizer.anonymize_message(messages[0]) if messages else None
+            },
+            'tool_calls': {
+                'count': len(tool_calls),
+                'sample_original': tool_calls[0] if tool_calls else None,
+                'sample_anonymized': self.anonymizer.anonymize_tool_call(tool_calls[0]) if tool_calls else None
+            }
+        }
+        
+        return preview
+    
+    def sync_conversation(self, local_logger, conversation_id: int, 
+                         anonymize: bool = True, dry_run: bool = False) -> Dict[str, Any]:
+        """
+        Sync a single conversation to the cloud.
+        
+        Args:
+            local_logger: ConversationLogger instance
+            conversation_id: ID of conversation to sync
+            anonymize: Whether to anonymize data (recommended)
+            dry_run: If True, don't actually upload, just show what would be sent
+        
+        Returns:
+            Result dictionary with success status and details
+        """
+        # Get conversation data
+        conversation = local_logger.get_conversation(conversation_id)
+        if not conversation:
+            return {'success': False, 'error': f'Conversation {conversation_id} not found'}
+        
+        messages = local_logger.get_messages(conversation_id)
+        tool_calls = local_logger.get_tool_calls(conversation_id)
+        
+        # Anonymize if requested
+        if anonymize:
+            conversation = self.anonymizer.anonymize_conversation(conversation)
+            messages = [self.anonymizer.anonymize_message(m) for m in messages]
+            tool_calls = [self.anonymizer.anonymize_tool_call(tc) for tc in tool_calls]
+        
+        # Prepare payload
+        payload = {
+            'conversation': conversation,
+            'messages': messages,
+            'tool_calls': tool_calls,
+            'anonymized': anonymize,
+            'sync_timestamp': datetime.utcnow().isoformat()
+        }
+        
+        if dry_run:
+            return {
+                'success': True,
+                'dry_run': True,
+                'payload_size': len(json.dumps(payload)),
+                'message_count': len(messages),
+                'tool_call_count': len(tool_calls),
+                'anonymized': anonymize
+            }
+        
+        # Upload to cloud
+        try:
+            headers = {'Content-Type': 'application/json'}
+            if self.api_key:
+                headers['Authorization'] = f'Bearer {self.api_key}'
+            
+            response = requests.post(
+                f'{self.api_url}/v1/training/upload',
+                json=payload,
+                headers=headers,
+                timeout=30
+            )
+            
+            response.raise_for_status()
+            
+            return {
+                'success': True,
+                'conversation_id': conversation_id,
+                'message_count': len(messages),
+                'tool_call_count': len(tool_calls),
+                'anonymized': anonymize,
+                'cloud_id': response.json().get('id')
+            }
+            
+        except requests.exceptions.RequestException as e:
+            return {
+                'success': False,
+                'error': str(e),
+                'conversation_id': conversation_id
+            }
+    
+    def sync_all_conversations(self, local_logger, anonymize: bool = True,
+                              dry_run: bool = False, 
+                              limit: Optional[int] = None) -> Dict[str, Any]:
+        """
+        Sync all conversations to the cloud.
+        
+        Args:
+            local_logger: ConversationLogger instance
+            anonymize: Whether to anonymize data
+            dry_run: If True, don't actually upload
+            limit: Optional limit on number of conversations to sync
+        
+        Returns:
+            Summary of sync operation
+        """
+        with local_logger._get_connection() as conn:
+            cursor = conn.cursor()
+            
+            query = "SELECT id FROM conversations ORDER BY started_at DESC"
+            if limit:
+                query += f" LIMIT {limit}"
+            
+            cursor.execute(query)
+            conversation_ids = [row['id'] for row in cursor.fetchall()]
+        
+        results = []
+        for conv_id in conversation_ids:
+            result = self.sync_conversation(local_logger, conv_id, anonymize, dry_run)
+            results.append(result)
+        
+        # Summarize
+        successful = sum(1 for r in results if r.get('success'))
+        failed = len(results) - successful
+        total_messages = sum(r.get('message_count', 0) for r in results)
+        total_tool_calls = sum(r.get('tool_call_count', 0) for r in results)
+        
+        return {
+            'success': failed == 0,
+            'total_conversations': len(results),
+            'successful': successful,
+            'failed': failed,
+            'total_messages': total_messages,
+            'total_tool_calls': total_tool_calls,
+            'anonymized': anonymize,
+            'dry_run': dry_run,
+            'results': results
+        }
+    
+    def get_sync_status(self) -> Dict[str, Any]:
+        """
+        Check cloud API status and sync health.
+        
+        Returns:
+            Status information
+        """
+        try:
+            headers = {}
+            if self.api_key:
+                headers['Authorization'] = f'Bearer {self.api_key}'
+            
+            response = requests.get(
+                f'{self.api_url}/v1/training/status',
+                headers=headers,
+                timeout=10
+            )
+            
+            response.raise_for_status()
+            return response.json()
+            
+        except requests.exceptions.RequestException as e:
+            return {
+                'success': False,
+                'error': str(e),
+                'api_url': self.api_url
+            }
+
+
+def create_sync_client(api_key: Optional[str] = None) -> CloudSync:
+    """
+    Create a cloud sync client with default API URL.
+    
+    Args:
+        api_key: Optional API key for authentication
+    
+    Returns:
+        CloudSync instance
+    """
+    # Default to production API
+    api_url = "https://brave-cliff-02c9bdd0f.3.azurestaticapps.net"
+    
+    # Or use environment variable
+    import os
+    api_url = os.environ.get('PYLANCE_MCP_API_URL', api_url)
+    
+    return CloudSync(api_url, api_key)