contextly 0.1.0__py3-none-any.whl

contextly/llm/manager.py

@@ -0,0 +1,126 @@
+"""
+LLM manager for code reasoning.
+"""
+
+import os
+from typing import Dict, Any, Optional, List, Union
+from pathlib import Path
+from .base import LLMProvider
+from .ollama import OllamaProvider
+from .openai import OpenAIProvider
+from .models import ModelManager, ModelProvider, ModelInfo
+
+class LLMManager:
+    """Manages LLM providers and generates code explanations."""
+    
+    def __init__(self, model: Optional[str] = None):
+        self.model_manager = ModelManager()
+        self.providers: Dict[str, LLMProvider] = {}
+        self.current_model = model or os.getenv('CONTEXTLY_MODEL', 'codellama')
+        self._initialize_providers()
+        
+    def _initialize_providers(self) -> None:
+        """Initialize LLM providers."""
+        # Add Ollama provider
+        self.providers['ollama'] = OllamaProvider()
+        
+        # Add OpenAI if key is available
+        if os.getenv('OPENAI_API_KEY'):
+            self.providers['openai'] = OpenAIProvider()
+            
+    def get_available_provider(self) -> Optional[LLMProvider]:
+        """Get the appropriate provider for the current model."""
+        model_info = self.model_manager.registry.get_model(self.current_model)
+        if not model_info:
+            # Try to download the model
+            if self.model_manager.download_model(self.current_model, ModelProvider.OLLAMA):
+                model_info = self.model_manager.registry.get_model(self.current_model)
+            
+        if model_info:
+            provider = self.providers.get(model_info.provider)
+            if provider and provider.is_available():
+                return provider
+                
+        # Fallback to any available provider
+        for provider in self.providers.values():
+            if provider.is_available():
+                return provider
+                
+        return None
+        
+    def list_models(self) -> List[Dict[str, Any]]:
+        """List all available models."""
+        return self.model_manager.list_available_models()
+        
+    def set_model(self, model_name: str) -> bool:
+        """Set the current model."""
+        if model_name not in [m['name'] for m in self.list_models()]:
+            # Try to download the model
+            if not self.model_manager.download_model(model_name, ModelProvider.OLLAMA):
+                return False
+        self.current_model = model_name
+        return True
+        
+    def explain_code(self, code: str, context: Optional[Dict[str, Any]] = None) -> str:
+        """Generate an explanation for a code snippet."""
+        provider = self.get_available_provider()
+        if not provider:
+            raise RuntimeError("No LLM provider available")
+            
+        # Build the prompt
+        prompt = self._build_explanation_prompt(code, context)
+        
+        # Generate the explanation
+        return provider.generate_response(prompt)
+        
+    def compare_configs(self, config1: Dict[str, Any], config2: Dict[str, Any]) -> str:
+        """Compare two configurations and explain differences."""
+        provider = self.get_available_provider()
+        if not provider:
+            raise RuntimeError("No LLM provider available")
+            
+        # Build the prompt
+        prompt = self._build_comparison_prompt(config1, config2)
+        
+        # Generate the comparison
+        return provider.generate_response(prompt)
+        
+    def _build_explanation_prompt(self, code: str, context: Optional[Dict[str, Any]]) -> str:
+        """Build a prompt for code explanation."""
+        prompt = [
+            "Please explain the following code in a clear and concise way.",
+            "Focus on the main purpose and key functionality.",
+            "\nCode to explain:\n```",
+            code,
+            "```\n"
+        ]
+        
+        if context:
+            prompt.extend([
+                "\nAdditional context:",
+                f"File: {context.get('file_path', 'unknown')}",
+                f"Type: {context.get('type', 'unknown')}",
+                f"Language: {context.get('language', 'unknown')}"
+            ])
+            
+        return "\n".join(prompt)
+        
+    def _build_comparison_prompt(self, config1: Dict[str, Any], config2: Dict[str, Any]) -> str:
+        """Build a prompt for configuration comparison."""
+        return f"""Please compare these two configurations and explain key differences and their potential impact:
+
+Configuration 1:
+```
+{config1}
+```
+
+Configuration 2:
+```
+{config2}
+```
+
+Please focus on:
+1. Missing or different keys
+2. Value type mismatches
+3. Potential issues or risks
+4. Recommended actions"""

contextly/llm/models.py

@@ -0,0 +1,204 @@
+"""
+Model management system for various LLM providers.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, Any, Optional, List
+import requests
+import json
+from pathlib import Path
+
+class ModelProvider(str, Enum):
+    """Supported model providers."""
+    OLLAMA = "ollama"
+    OPENAI = "openai"
+    HUGGINGFACE = "huggingface"
+    LOCALAI = "localai"
+    CUSTOM = "custom"
+
+@dataclass
+class ModelInfo:
+    """Information about an LLM model."""
+    name: str
+    provider: ModelProvider
+    description: str
+    context_length: int
+    quantization: Optional[str] = None
+    metadata: Dict[str, Any] = None
+
+class ModelRegistry:
+    """Registry of available models and their capabilities."""
+    
+    def __init__(self, config_path: Optional[Path] = None):
+        self.config_path = config_path or Path.home() / ".contextly" / "models.json"
+        self.models: Dict[str, ModelInfo] = {}
+        self._load_models()
+        
+    def _load_models(self) -> None:
+        """Load model registry from config file."""
+        if self.config_path.exists():
+            with open(self.config_path, 'r') as f:
+                data = json.load(f)
+                for model_data in data.get('models', []):
+                    info = ModelInfo(
+                        name=model_data['name'],
+                        provider=ModelProvider(model_data['provider']),
+                        description=model_data.get('description', ''),
+                        context_length=model_data.get('context_length', 4096),
+                        quantization=model_data.get('quantization'),
+                        metadata=model_data.get('metadata', {})
+                    )
+                    self.models[model_data['name']] = info
+                    
+    def _save_models(self) -> None:
+        """Save model registry to config file."""
+        self.config_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.config_path, 'w') as f:
+            json.dump({
+                'models': [
+                    {
+                        'name': info.name,
+                        'provider': info.provider,
+                        'description': info.description,
+                        'context_length': info.context_length,
+                        'quantization': info.quantization,
+                        'metadata': info.metadata
+                    }
+                    for info in self.models.values()
+                ]
+            }, f, indent=2)
+            
+    def register_model(self, model_info: ModelInfo) -> None:
+        """Register a new model."""
+        self.models[model_info.name] = model_info
+        self._save_models()
+        
+    def get_model(self, name: str) -> Optional[ModelInfo]:
+        """Get information about a specific model."""
+        return self.models.get(name)
+        
+    def list_models(self, provider: Optional[ModelProvider] = None) -> List[ModelInfo]:
+        """List all registered models, optionally filtered by provider."""
+        if provider:
+            return [m for m in self.models.values() if m.provider == provider]
+        return list(self.models.values())
+        
+    def remove_model(self, name: str) -> bool:
+        """Remove a model from the registry."""
+        if name in self.models:
+            del self.models[name]
+            self._save_models()
+            return True
+        return False
+
+class ModelManager:
+    """Manages model downloads and updates."""
+    
+    OLLAMA_API = "http://localhost:11434/api"
+    HF_API = "https://huggingface.co/api"
+    
+    def __init__(self):
+        self.registry = ModelRegistry()
+        
+    def download_model(self, name: str, provider: ModelProvider, **kwargs) -> bool:
+        """Download a model from its provider."""
+        try:
+            if provider == ModelProvider.OLLAMA:
+                return self._download_ollama_model(name)
+            elif provider == ModelProvider.HUGGINGFACE:
+                return self._download_hf_model(name, **kwargs)
+            elif provider == ModelProvider.CUSTOM:
+                return self._download_custom_model(name, **kwargs)
+            else:
+                raise ValueError(f"Unsupported provider: {provider}")
+        except Exception as e:
+            print(f"Failed to download model {name}: {e}")
+            return False
+            
+    def _download_ollama_model(self, name: str) -> bool:
+        """Download a model from Ollama."""
+        try:
+            response = requests.post(
+                f"{self.OLLAMA_API}/pull",
+                json={"name": name}
+            )
+            if response.status_code == 200:
+                # Register model
+                self.registry.register_model(ModelInfo(
+                    name=name,
+                    provider=ModelProvider.OLLAMA,
+                    description=f"Ollama model: {name}",
+                    context_length=8192  # Default for most Ollama models
+                ))
+                return True
+            return False
+        except Exception:
+            return False
+            
+    def _download_hf_model(self, name: str, token: Optional[str] = None) -> bool:
+        """Download a model from Hugging Face."""
+        headers = {"Authorization": f"Bearer {token}"} if token else {}
+        try:
+            # Get model info
+            response = requests.get(
+                f"{self.HF_API}/models/{name}",
+                headers=headers
+            )
+            if response.status_code == 200:
+                model_info = response.json()
+                self.registry.register_model(ModelInfo(
+                    name=name,
+                    provider=ModelProvider.HUGGINGFACE,
+                    description=model_info.get('description', ''),
+                    context_length=model_info.get('max_position_embeddings', 4096),
+                    metadata=model_info
+                ))
+                return True
+            return False
+        except Exception:
+            return False
+            
+    def _download_custom_model(self, name: str, url: str, **kwargs) -> bool:
+        """Download a custom model from a URL."""
+        try:
+            self.registry.register_model(ModelInfo(
+                name=name,
+                provider=ModelProvider.CUSTOM,
+                description=kwargs.get('description', f"Custom model: {name}"),
+                context_length=kwargs.get('context_length', 4096),
+                metadata={'url': url, **kwargs}
+            ))
+            return True
+        except Exception:
+            return False
+            
+    def list_available_models(self) -> List[Dict[str, Any]]:
+        """List all available models from different providers."""
+        models = []
+        
+        # Get Ollama models
+        try:
+            response = requests.get(f"{self.OLLAMA_API}/tags")
+            if response.status_code == 200:
+                models.extend([
+                    {
+                        'name': model['name'],
+                        'provider': 'ollama',
+                        'status': 'available'
+                    }
+                    for model in response.json().get('models', [])
+                ])
+        except Exception:
+            pass
+            
+        # Add registered models
+        for model in self.registry.list_models():
+            models.append({
+                'name': model.name,
+                'provider': model.provider,
+                'description': model.description,
+                'status': 'registered'
+            })
+            
+        return models

contextly/llm/ollama.py

@@ -0,0 +1,73 @@
+"""
+Integration with local Ollama models.
+"""
+
+import json
+import requests
+from typing import Dict, Any, Optional
+from .base import LLMProvider
+
+class OllamaProvider(LLMProvider):
+    """LLM provider using local Ollama models."""
+    
+    DEFAULT_MODEL = "codellama"
+    BASE_URL = "http://localhost:11434/api"
+    
+    def __init__(self, model: str = DEFAULT_MODEL):
+        self.model = model
+        self._ensure_model()
+        
+    def _ensure_model(self) -> None:
+        """Ensure the model is downloaded and ready."""
+        try:
+            # Check if model exists
+            response = requests.get(f"{self.BASE_URL}/tags")
+            if response.status_code == 200:
+                models = response.json().get("models", [])
+                if not any(model["name"] == self.model for model in models):
+                    print(f"Downloading {self.model}...")
+                    # Pull the model
+                    pull_response = requests.post(
+                        f"{self.BASE_URL}/pull",
+                        json={"name": self.model}
+                    )
+                    if pull_response.status_code != 200:
+                        raise RuntimeError(f"Failed to pull model: {pull_response.text}")
+                    print(f"{self.model} ready!")
+        except Exception as e:
+            raise RuntimeError(f"Failed to set up Ollama: {str(e)}")
+    
+    def is_available(self) -> bool:
+        """Check if Ollama is running and model is available."""
+        try:
+            response = requests.get(f"{self.BASE_URL}/tags")
+            return response.status_code == 200
+        except:
+            return False
+            
+    def generate_response(self, prompt: str, **kwargs) -> str:
+        """Generate a response using the Ollama model."""
+        try:
+            # Set up optimal parameters for code
+            params = {
+                "model": self.model,
+                "prompt": prompt,
+                "stream": False,
+                "temperature": kwargs.get("temperature", 0.2),  # Lower temp for code
+                "top_p": kwargs.get("top_p", 0.9),
+                "system": "You are an expert programmer helping to understand and explain code.",
+                **kwargs
+            }
+            
+            response = requests.post(
+                f"{self.BASE_URL}/generate",
+                json=params
+            )
+            
+            if response.status_code == 200:
+                return response.json()["response"]
+            else:
+                raise RuntimeError(f"Ollama API error: {response.status_code} - {response.text}")
+                
+        except Exception as e:
+            raise RuntimeError(f"Failed to generate response: {str(e)}")

contextly/llm/openai.py

@@ -0,0 +1,39 @@
+"""
+Integration with OpenAI API as a fallback.
+"""
+
+import os
+from typing import Dict, Any, Optional
+from openai import OpenAI
+from .base import LLMProvider
+
+class OpenAIProvider(LLMProvider):
+    """LLM provider using OpenAI API."""
+    
+    DEFAULT_MODEL = "gpt-4-mini"
+    
+    def __init__(self, model: str = DEFAULT_MODEL, api_key: Optional[str] = None):
+        self.model = model
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        self.client = OpenAI(api_key=self.api_key) if self.api_key else None
+        
+    def is_available(self) -> bool:
+        """Check if OpenAI API is configured."""
+        return bool(self.client and self.api_key)
+        
+    def generate_response(self, prompt: str, **kwargs) -> str:
+        """Generate a response using the OpenAI API."""
+        if not self.is_available():
+            raise RuntimeError("OpenAI API is not configured")
+            
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=[{"role": "user", "content": prompt}],
+                **kwargs
+            )
+            
+            return response.choices[0].message.content.strip()
+            
+        except Exception as e:
+            raise RuntimeError(f"OpenAI API error: {str(e)}")

contextly/parsers/base.py

@@ -0,0 +1,39 @@
+"""
+Base parser interface and utilities.
+"""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+
+class BaseParser(ABC):
+    """Abstract base class for all file parsers."""
+    
+    def __init__(self):
+        self.supported_extensions: List[str] = []
+        
+    @abstractmethod
+    def parse(self, file_path: Path) -> Dict[str, Any]:
+        """Parse a file and return structured content."""
+        pass
+        
+    def can_parse(self, file_path: Path) -> bool:
+        """Check if this parser can handle the given file."""
+        return file_path.suffix in self.supported_extensions
+        
+class ChunkMetadata:
+    """Metadata for a code or text chunk."""
+    
+    def __init__(
+        self,
+        file_path: Path,
+        start_line: int,
+        end_line: int,
+        content_type: str,
+        symbols: Optional[List[str]] = None
+    ):
+        self.file_path = file_path
+        self.start_line = start_line
+        self.end_line = end_line
+        self.content_type = content_type
+        self.symbols = symbols or []

contextly/parsers/config.py

@@ -0,0 +1,79 @@
+"""
+Parser for configuration files (JSON, YAML, T                'file_type': 'env' if str(file_path).startswith('.env') else self.file_type_map.get(str(file_path).split('.')[-1], 'unknown')ML, ENV).
+"""
+
+import json
+import yaml
+import toml
+from pathlib import Path
+from typing import Dict, Any
+from dotenv import dotenv_values
+from .base import BaseParser
+
+class ConfigParser(BaseParser):
+    """Parser for various configuration file formats."""
+    
+    def __init__(self):
+        super().__init__()
+        self.supported_extensions = ['.json', '.yml', '.yaml', '.toml', '.env']
+        self.file_type_map = {
+            'env': 'env',
+            'json': 'json',
+            'yml': 'yaml',
+            'yaml': 'yaml',
+            'toml': 'toml'
+        }
+        
+    def parse(self, file_path: Path) -> Dict[str, Any]:
+        """Parse configuration file based on its extension."""
+        ext = file_path.suffix
+        
+        try:
+            # Read file content first
+            with open(file_path, 'r') as f:
+                content = f.read().strip()
+
+            parsed: Dict[str, Any] = {}
+            # Parse .env files
+            if file_path.name.startswith('.env'):
+                for line in content.split('\n'):
+                    line = line.strip()
+                    if line and not line.startswith('#'):
+                        key, value = line.split('=', 1)
+                        parsed[key.strip()] = value.strip()
+            
+            if ext == '.json':
+                parsed = json.loads(content)
+            elif ext in ['.yml', '.yaml']:
+                parsed = yaml.safe_load(content)
+            elif ext == '.toml':
+                parsed = toml.loads(content)
+            elif ext == '.env':
+                parsed = dict(dotenv_values(file_path))
+                
+            # Only parse non-env files through their respective parsers
+            if not file_path.name.startswith('.env'):
+                if ext == '.json':
+                    parsed = json.loads(content)
+                elif ext in ['.yml', '.yaml']:
+                    parsed = yaml.safe_load(content)
+                elif ext == '.toml':
+                    parsed = toml.loads(content)
+
+            return {
+                'file_type': 'env' if file_path.name.startswith('.env') else file_path.suffix.lstrip('.').lower(),
+                'content': content,
+                'parsed': {k.strip(): v.strip() for k, v in parsed.items()},  # Clean up parsed values
+                'chunks': [{
+                    'type': 'config',
+                    'content': content,
+                    'parsed': parsed
+                }]
+            }
+                
+        except Exception as e:
+            return {
+                'error': f'Failed to parse {file_path}: {str(e)}',
+                'content': content if 'content' in locals() else '',
+                'chunks': []
+            }

contextly/parsers/javascript.py

@@ -0,0 +1,122 @@
+"""
+Parser for JavaScript and TypeScript files.
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Match
+from .base import BaseParser
+
+class JavaScriptParser(BaseParser):
+    """Parser for JavaScript and TypeScript files using regex patterns."""
+    
+    def __init__(self):
+        super().__init__()
+        self.supported_extensions = ['.js', '.ts', '.jsx', '.tsx']
+        
+        # Regular expressions for parsing
+        self.patterns = {
+            'function': re.compile(
+                r'(?:export\s+)?'  # Optional export
+                r'(?:async\s+)?'   # Optional async
+                r'function\s+'  # Function keyword is required for top-level functions
+                r'(\w+)'           # Function name
+                r'\s*'
+                r'\([^)]*\)'       # Parameters
+                r'\s*{',          # Opening brace
+                re.MULTILINE
+            ),
+            'class': re.compile(
+                r'(?:export\s+)?'  # Optional export
+                r'class\s+'        # Class keyword
+                r'(\w+)'           # Class name
+                r'(?:\s+extends\s+\w+)?'  # Optional inheritance
+                r'\s*{',          # Opening brace
+                re.MULTILINE
+            ),
+            'import': re.compile(
+                r'import\s+'
+                r'(?:{[^}]+}|[^;]+)'  # Named imports or default import
+                r'\s+from\s+'
+                r'[\'"]([^\'"]+)[\'"]',  # Module path
+                re.MULTILINE
+            ),
+            'export': re.compile(
+                r'export\s+'
+                r'(?:{[^}]+}|[^;]+)',  # Named exports or default export
+                re.MULTILINE
+            )
+        }
+        
+    def _find_block_end(self, content: str, start: int) -> int:
+        """Find the end of a code block starting from a position."""
+        level = 0
+        for i in range(start, len(content)):
+            if content[i] == '{':
+                level += 1
+            elif content[i] == '}':
+                level -= 1
+                if level == 0:
+                    return i
+        return len(content)
+        
+    def _get_line_number(self, content: str, pos: int) -> int:
+        """Convert position to line number."""
+        return content.count('\n', 0, pos) + 1
+        
+    def _extract_code_block(self, content: str, match: Match[str], context_lines: int = 2) -> Dict[str, Any]:
+        """Extract a code block with surrounding context."""
+        start_pos = match.start()
+        block_end = self._find_block_end(content, start_pos)
+        
+        # Get line numbers
+        start_line = self._get_line_number(content, start_pos)
+        end_line = self._get_line_number(content, block_end)
+        
+        # Extract the block with context
+        lines = content.split('\n')
+        context_start = max(0, start_line - context_lines - 1)
+        context_end = min(len(lines), end_line + context_lines)
+        block_content = '\n'.join(lines[context_start:context_end])
+        
+        return {
+            'start_line': start_line,
+            'end_line': end_line,
+            'content': block_content
+        }
+        
+    def parse(self, file_path: Path) -> Dict[str, Any]:
+        """Parse JavaScript/TypeScript file and extract structure and symbols."""
+        with open(file_path, 'r') as f:
+            content = f.read()
+            
+        chunks: List[Dict[str, Any]] = []
+        symbols: List[str] = []
+        imports: List[str] = []
+        
+        # Find all patterns in the content
+        for pattern_name, pattern in self.patterns.items():
+            for match in pattern.finditer(content):
+                if pattern_name in ['function', 'class']:
+                    name = match.group(1)
+                    block_info = self._extract_code_block(content, match)
+                    
+                    chunks.append({
+                        'type': pattern_name,
+                        'name': name,
+                        'start_line': block_info['start_line'],
+                        'end_line': block_info['end_line'],
+                        'content': block_info['content'],
+                    })
+                    symbols.append(name)
+                    
+                elif pattern_name == 'import':
+                    imports.append(match.group(1))
+                    
+        return {
+            'file_type': 'javascript',  # Default type for all JS/TS files for now
+            'content': content,
+            'chunks': chunks,
+            'symbols': symbols,
+            'imports': imports
+        }