git-llm-tool 0.1.12__py3-none-any.whl

git_llm_tool/core/config.py

@@ -0,0 +1,352 @@
+"""Configuration management for git-llm-tool."""
+
+import os
+import yaml
+from pathlib import Path
+from typing import Dict, Any, Optional
+from dataclasses import dataclass, field
+
+from git_llm_tool.core.exceptions import ConfigError
+
+
+@dataclass
+class LlmConfig:
+    """LLM configuration settings."""
+    default_model: str = "gpt-4o"
+    language: str = "en"
+    api_keys: Dict[str, str] = field(default_factory=dict)
+    azure_openai: Dict[str, str] = field(default_factory=dict)  # endpoint, api_version, deployment_name
+
+    # Processing configuration
+    chunking_threshold: int = 12000  # Token threshold to trigger chunking + parallel processing
+
+    # Ollama configuration for hybrid processing
+    use_ollama_for_chunks: bool = False  # Use Ollama for chunk processing (map phase)
+    ollama_model: str = "llama3:8b"  # Ollama model for chunk processing
+    ollama_base_url: str = "http://localhost:11434"  # Ollama API base URL
+
+    # Parallel processing configuration
+    max_parallel_chunks: int = 4  # Maximum concurrent chunks for remote APIs
+    ollama_max_parallel_chunks: int = 16  # Maximum concurrent chunks for Ollama (local)
+
+    # Internal constants (not user-configurable)
+    _chunk_size: int = 6000  # Maximum chunk size in characters
+    _chunk_overlap: int = 300  # Overlap between chunks to maintain context
+    _max_parallel_chunks: int = 4  # Maximum number of chunks to process in parallel (remote APIs)
+    _ollama_max_parallel_chunks: int = 16  # Maximum number of chunks to process in parallel (Ollama local)
+    _chunk_processing_timeout: float = 120.0  # Timeout for each chunk processing (seconds)
+    _max_retries: int = 5  # Maximum number of retries
+    _initial_delay: float = 1.0  # Initial retry delay in seconds
+    _max_delay: float = 60.0  # Maximum retry delay in seconds
+    _backoff_multiplier: float = 2.0  # Exponential backoff multiplier
+    _rate_limit_delay: float = 0.5  # Minimum delay between requests
+    _max_context_lines: int = 3  # Maximum context lines to keep
+    _max_tokens: int = 8000  # Maximum tokens before truncation
+
+
+@dataclass
+class JiraConfig:
+    """Jira integration configuration."""
+    enabled: bool = False
+    ticket_pattern: Optional[str] = None  # Jira ticket regex pattern
+
+
+@dataclass
+class EditorConfig:
+    """Editor configuration settings."""
+    preferred_editor: Optional[str] = None  # e.g., "vi", "nano", "code", etc.
+
+
+@dataclass
+class AppConfig:
+    """Main application configuration."""
+    llm: LlmConfig = field(default_factory=LlmConfig)
+    jira: JiraConfig = field(default_factory=JiraConfig)
+    editor: EditorConfig = field(default_factory=EditorConfig)
+
+
+class ConfigLoader:
+    """Singleton configuration loader with hierarchical configuration support."""
+
+    _instance = None
+    _config = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if not getattr(self, '_initialized', False):
+            self._config = self._load_config()
+            self._initialized = True
+
+    @property
+    def config(self) -> AppConfig:
+        """Get the loaded configuration."""
+        return self._config
+
+    def _load_config(self) -> AppConfig:
+        """Load configuration from multiple sources in hierarchical order."""
+        config_data = {}
+
+        # 1. Load global config
+        global_config_path = Path.home() / ".git-llm-tool" / "config.yaml"
+        if global_config_path.exists():
+            config_data.update(self._load_yaml_file(global_config_path))
+
+        # 2. Load project config (override global)
+        project_config_path = Path(".git-llm-tool.yaml")
+        if project_config_path.exists():
+            project_config = self._load_yaml_file(project_config_path)
+            config_data = self._merge_configs(config_data, project_config)
+
+        # 3. Load environment variables (override file configs)
+        env_config = self._load_env_config()
+        config_data = self._merge_configs(config_data, env_config)
+
+        return self._create_app_config(config_data)
+
+    def _load_yaml_file(self, file_path: Path) -> Dict[str, Any]:
+        """Load YAML configuration file."""
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = yaml.safe_load(f)
+                return data if data is not None else {}
+        except yaml.YAMLError as e:
+            raise ConfigError(f"Invalid YAML in {file_path}: {e}")
+        except Exception as e:
+            raise ConfigError(f"Failed to read config file {file_path}: {e}")
+
+    def _load_env_config(self) -> Dict[str, Any]:
+        """Load configuration from environment variables."""
+        config = {}
+
+        # API keys from environment
+        api_keys = {}
+        if openai_key := os.getenv("OPENAI_API_KEY"):
+            api_keys["openai"] = openai_key
+        if anthropic_key := os.getenv("ANTHROPIC_API_KEY"):
+            api_keys["anthropic"] = anthropic_key
+        if google_key := os.getenv("GOOGLE_API_KEY"):
+            api_keys["google"] = google_key
+
+        # Azure OpenAI configuration from environment
+        azure_openai = {}
+        if azure_endpoint := os.getenv("AZURE_OPENAI_ENDPOINT"):
+            azure_openai["endpoint"] = azure_endpoint
+        if azure_key := os.getenv("AZURE_OPENAI_API_KEY"):
+            api_keys["azure_openai"] = azure_key
+        if azure_version := os.getenv("AZURE_OPENAI_API_VERSION"):
+            azure_openai["api_version"] = azure_version
+        if azure_deployment := os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"):
+            azure_openai["deployment_name"] = azure_deployment
+
+        # Set up LLM config
+        if api_keys or azure_openai:
+            config["llm"] = {}
+            if api_keys:
+                config["llm"]["api_keys"] = api_keys
+            if azure_openai:
+                config["llm"]["azure_openai"] = azure_openai
+
+        # Other environment variables
+        if model := os.getenv("GIT_LLM_MODEL"):
+            config.setdefault("llm", {})["default_model"] = model
+
+        if language := os.getenv("GIT_LLM_LANGUAGE"):
+            config.setdefault("llm", {})["language"] = language
+
+        return config
+
+    def _merge_configs(self, base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
+        """Merge two configuration dictionaries recursively."""
+        result = base.copy()
+
+        for key, value in override.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._merge_configs(result[key], value)
+            else:
+                result[key] = value
+
+        return result
+
+    def _create_app_config(self, config_data: Dict[str, Any]) -> AppConfig:
+        """Create AppConfig instance from configuration data."""
+        # Create LLM config
+        llm_data = config_data.get("llm", {})
+        llm_config = LlmConfig(
+            default_model=llm_data.get("default_model", "gpt-4o"),
+            language=llm_data.get("language", "en"),
+            api_keys=llm_data.get("api_keys", {}),
+            azure_openai=llm_data.get("azure_openai", {}),
+            # Processing settings
+            chunking_threshold=llm_data.get("chunking_threshold", 12000),
+            # Ollama settings
+            use_ollama_for_chunks=llm_data.get("use_ollama_for_chunks", False),
+            ollama_model=llm_data.get("ollama_model", "llama3:8b"),
+            ollama_base_url=llm_data.get("ollama_base_url", "http://localhost:11434")
+        )
+
+        # Create Jira config
+        jira_data = config_data.get("jira", {})
+        jira_config = JiraConfig(
+            enabled=jira_data.get("enabled", False),
+            ticket_pattern=jira_data.get("ticket_pattern")
+        )
+
+        # Create Editor config
+        editor_data = config_data.get("editor", {})
+        editor_config = EditorConfig(
+            preferred_editor=editor_data.get("preferred_editor")
+        )
+
+        return AppConfig(llm=llm_config, jira=jira_config, editor=editor_config)
+
+    def save_config(self, config_path: Optional[Path] = None) -> None:
+        """Save current configuration to file."""
+        if config_path is None:
+            # Save to global config by default
+            config_path = Path.home() / ".git-llm-tool" / "config.yaml"
+
+        # Ensure directory exists
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Convert config to dict
+        config_dict = {
+            "llm": {
+                "default_model": self._config.llm.default_model,
+                "language": self._config.llm.language,
+                "api_keys": self._config.llm.api_keys,
+                "azure_openai": self._config.llm.azure_openai,
+                "chunking_threshold": self._config.llm.chunking_threshold,
+                "use_ollama_for_chunks": self._config.llm.use_ollama_for_chunks,
+                "ollama_model": self._config.llm.ollama_model,
+                "ollama_base_url": self._config.llm.ollama_base_url
+            },
+            "jira": {
+                "enabled": self._config.jira.enabled,
+                "ticket_pattern": self._config.jira.ticket_pattern
+            },
+            "editor": {
+                "preferred_editor": self._config.editor.preferred_editor
+            }
+        }
+
+        # Remove empty sections to keep config clean
+        if not config_dict["llm"]["api_keys"]:
+            del config_dict["llm"]["api_keys"]
+        if not config_dict["llm"]["azure_openai"]:
+            del config_dict["llm"]["azure_openai"]
+
+        # Remove None values from jira config
+        if config_dict["jira"]["ticket_pattern"] is None:
+            del config_dict["jira"]["ticket_pattern"]
+
+        # Remove None values from editor config
+        if config_dict["editor"]["preferred_editor"] is None:
+            del config_dict["editor"]["preferred_editor"]
+        if not config_dict["editor"]:
+            del config_dict["editor"]
+
+        try:
+            with open(config_path, 'w', encoding='utf-8') as f:
+                yaml.dump(config_dict, f, default_flow_style=False, indent=2)
+        except Exception as e:
+            raise ConfigError(f"Failed to save config to {config_path}: {e}")
+
+    def set_value(self, key_path: str, value: str) -> None:
+        """Set a configuration value using dot notation (e.g., 'llm.default_model')."""
+        keys = key_path.split('.')
+
+        if len(keys) < 2:
+            raise ConfigError(f"Invalid key path: {key_path}")
+
+        # Handle llm.default_model
+        if keys[0] == "llm" and keys[1] == "default_model":
+            self._config.llm.default_model = value
+        # Handle llm.language
+        elif keys[0] == "llm" and keys[1] == "language":
+            self._config.llm.language = value
+        # Handle llm.use_ollama_for_chunks
+        elif keys[0] == "llm" and keys[1] == "use_ollama_for_chunks":
+            self._config.llm.use_ollama_for_chunks = value.lower() in ("true", "1", "yes", "on")
+        # Handle llm.ollama_model
+        elif keys[0] == "llm" and keys[1] == "ollama_model":
+            self._config.llm.ollama_model = value
+        # Handle llm.ollama_base_url
+        elif keys[0] == "llm" and keys[1] == "ollama_base_url":
+            self._config.llm.ollama_base_url = value
+        # Handle llm.api_keys.*
+        elif keys[0] == "llm" and keys[1] == "api_keys" and len(keys) == 3:
+            self._config.llm.api_keys[keys[2]] = value
+        # Handle llm.azure_openai.*
+        elif keys[0] == "llm" and keys[1] == "azure_openai" and len(keys) == 3:
+            self._config.llm.azure_openai[keys[2]] = value
+        # Handle jira.enabled
+        elif keys[0] == "jira" and keys[1] == "enabled":
+            self._config.jira.enabled = value.lower() in ("true", "1", "yes", "on")
+        # Handle jira.ticket_pattern
+        elif keys[0] == "jira" and keys[1] == "ticket_pattern":
+            self._config.jira.ticket_pattern = value
+        # Handle editor.preferred_editor
+        elif keys[0] == "editor" and keys[1] == "preferred_editor":
+            self._config.editor.preferred_editor = value
+        else:
+            raise ConfigError(f"Unknown configuration key: {key_path}")
+
+    def get_value(self, key_path: str) -> Any:
+        """Get a configuration value using dot notation."""
+        keys = key_path.split('.')
+
+        if len(keys) < 2:
+            raise ConfigError(f"Invalid key path: {key_path}")
+
+        # Handle llm.default_model
+        if keys[0] == "llm" and keys[1] == "default_model":
+            return self._config.llm.default_model
+        # Handle llm.language
+        elif keys[0] == "llm" and keys[1] == "language":
+            return self._config.llm.language
+        # Handle llm.use_ollama_for_chunks
+        elif keys[0] == "llm" and keys[1] == "use_ollama_for_chunks":
+            return self._config.llm.use_ollama_for_chunks
+        # Handle llm.ollama_model
+        elif keys[0] == "llm" and keys[1] == "ollama_model":
+            return self._config.llm.ollama_model
+        # Handle llm.ollama_base_url
+        elif keys[0] == "llm" and keys[1] == "ollama_base_url":
+            return self._config.llm.ollama_base_url
+        # Handle llm.api_keys.*
+        elif keys[0] == "llm" and keys[1] == "api_keys" and len(keys) == 3:
+            return self._config.llm.api_keys.get(keys[2])
+        # Handle llm.azure_openai.*
+        elif keys[0] == "llm" and keys[1] == "azure_openai" and len(keys) == 3:
+            return self._config.llm.azure_openai.get(keys[2])
+        # Handle jira.enabled
+        elif keys[0] == "jira" and keys[1] == "enabled":
+            return self._config.jira.enabled
+        # Handle jira.ticket_pattern
+        elif keys[0] == "jira" and keys[1] == "ticket_pattern":
+            return self._config.jira.ticket_pattern
+        # Handle editor.preferred_editor
+        elif keys[0] == "editor" and keys[1] == "preferred_editor":
+            return self._config.editor.preferred_editor
+        else:
+            raise ConfigError(f"Unknown configuration key: {key_path}")
+
+    def reload(self) -> None:
+        """Reload configuration from files."""
+        self._config = self._load_config()
+
+    @classmethod
+    def _reset_instance(cls) -> None:
+        """Reset singleton instance for testing."""
+        cls._instance = None
+        cls._config = None
+
+
+def get_config() -> AppConfig:
+    """Get the application configuration."""
+    return ConfigLoader().config

git_llm_tool/core/diff_optimizer.py

@@ -0,0 +1,206 @@
+"""Diff optimization strategies to reduce token usage."""
+
+import re
+from typing import List, Tuple, Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class DiffStats:
+    """Statistics about diff optimization."""
+    original_size: int
+    optimized_size: int
+    compression_ratio: float
+    files_processed: int
+    lines_removed: int
+
+
+class DiffOptimizer:
+    """Optimize git diffs to reduce token usage while preserving meaning."""
+
+    def __init__(self, max_context_lines: int = 3):
+        self.max_context_lines = max_context_lines
+
+    def optimize_diff(self, diff: str, aggressive: bool = False) -> Tuple[str, DiffStats]:
+        """Optimize diff to reduce token usage."""
+        original_size = len(diff)
+        lines = diff.split('\n')
+
+        optimized_lines = []
+        files_processed = 0
+        lines_removed = 0
+
+        i = 0
+        while i < len(lines):
+            line = lines[i]
+
+            # Keep file headers
+            if line.startswith('diff --git'):
+                files_processed += 1
+                optimized_lines.append(line)
+                i += 1
+                continue
+
+            # Keep index and mode lines (compressed)
+            if line.startswith('index ') or line.startswith('new file mode') or line.startswith('deleted file mode'):
+                if not aggressive:
+                    optimized_lines.append(line)
+                else:
+                    lines_removed += 1
+                i += 1
+                continue
+
+            # Keep file paths but simplify
+            if line.startswith('--- ') or line.startswith('+++ '):
+                if aggressive:
+                    # Simplify path names
+                    simplified = re.sub(r'^(---|\+\+\+) [ab]/', r'\1 ', line)
+                    optimized_lines.append(simplified)
+                else:
+                    optimized_lines.append(line)
+                i += 1
+                continue
+
+            # Process hunk headers
+            if line.startswith('@@'):
+                optimized_lines.append(line)
+                i += 1
+
+                # Process the content of this hunk
+                hunk_lines, removed_count = self._process_hunk(lines[i:], aggressive)
+                optimized_lines.extend(hunk_lines)
+                lines_removed += removed_count
+                i += len(hunk_lines)
+                continue
+
+            # Keep other lines as-is
+            optimized_lines.append(line)
+            i += 1
+
+        optimized_diff = '\n'.join(optimized_lines)
+        optimized_size = len(optimized_diff)
+
+        stats = DiffStats(
+            original_size=original_size,
+            optimized_size=optimized_size,
+            compression_ratio=optimized_size / original_size if original_size > 0 else 1.0,
+            files_processed=files_processed,
+            lines_removed=lines_removed
+        )
+
+        return optimized_diff, stats
+
+    def _process_hunk(self, lines: List[str], aggressive: bool) -> Tuple[List[str], int]:
+        """Process a hunk to optimize context and changes."""
+        result = []
+        removed_count = 0
+
+        # Group consecutive context lines
+        context_buffer = []
+
+        for line in lines:
+            # Stop at next hunk or file
+            if line.startswith('@@') or line.startswith('diff --git'):
+                break
+
+            if line.startswith(' '):  # Context line
+                context_buffer.append(line)
+            else:  # Added or removed line
+                # Flush context buffer with limit
+                if context_buffer:
+                    if aggressive and len(context_buffer) > self.max_context_lines * 2:
+                        # Keep first and last few context lines
+                        keep_start = context_buffer[:self.max_context_lines]
+                        keep_end = context_buffer[-self.max_context_lines:]
+                        result.extend(keep_start)
+                        if len(context_buffer) > self.max_context_lines * 2:
+                            result.append(f' ... ({len(context_buffer) - len(keep_start) - len(keep_end)} context lines omitted)')
+                        result.extend(keep_end)
+                        removed_count += len(context_buffer) - len(keep_start) - len(keep_end)
+                    else:
+                        result.extend(context_buffer)
+                    context_buffer = []
+
+                # Keep change lines (but can compress whitespace-only changes)
+                if aggressive and self._is_whitespace_only_change(line):
+                    result.append(f'{line[0]} (whitespace change)')
+                    removed_count += 1
+                else:
+                    result.append(line)
+
+        # Handle remaining context
+        if context_buffer:
+            if aggressive and len(context_buffer) > self.max_context_lines:
+                result.extend(context_buffer[:self.max_context_lines])
+                result.append(f' ... ({len(context_buffer) - self.max_context_lines} trailing context lines omitted)')
+                removed_count += len(context_buffer) - self.max_context_lines
+            else:
+                result.extend(context_buffer)
+
+        return result, removed_count
+
+    def _is_whitespace_only_change(self, line: str) -> bool:
+        """Check if a line represents only whitespace changes."""
+        if len(line) < 2:
+            return False
+
+        content = line[1:]  # Remove +/- prefix
+        return content.strip() == '' or re.match(r'^\s+$', content)
+
+    def smart_truncate(self, diff: str, max_tokens: int = 8000) -> str:
+        """Smart truncation that preserves important parts of the diff."""
+        # Rough estimation: 1 token ≈ 4 characters
+        max_chars = max_tokens * 4
+
+        if len(diff) <= max_chars:
+            return diff
+
+        lines = diff.split('\n')
+        important_lines = []
+        regular_lines = []
+
+        for line in lines:
+            if self._is_important_line(line):
+                important_lines.append(line)
+            else:
+                regular_lines.append(line)
+
+        # Always keep important lines
+        result = important_lines[:]
+        current_size = sum(len(line) + 1 for line in result)  # +1 for newline
+
+        # Add regular lines until we hit the limit
+        for line in regular_lines:
+            if current_size + len(line) + 1 > max_chars:
+                result.append('... (diff truncated to fit token limit)')
+                break
+            result.append(line)
+            current_size += len(line) + 1
+
+        return '\n'.join(result)
+
+    def _is_important_line(self, line: str) -> bool:
+        """Determine if a line is important and should be preserved."""
+        return (
+            line.startswith('diff --git') or
+            line.startswith('+++') or
+            line.startswith('---') or
+            line.startswith('@@') or
+            (line.startswith('+') and not self._is_whitespace_only_change(line)) or
+            (line.startswith('-') and not self._is_whitespace_only_change(line))
+        )
+
+    def get_summary_stats(self, diff: str) -> dict:
+        """Get summary statistics about a diff."""
+        lines = diff.split('\n')
+
+        stats = {
+            'total_lines': len(lines),
+            'files_changed': len([l for l in lines if l.startswith('diff --git')]),
+            'lines_added': len([l for l in lines if l.startswith('+') and not l.startswith('+++')]),
+            'lines_removed': len([l for l in lines if l.startswith('-') and not l.startswith('---')]),
+            'context_lines': len([l for l in lines if l.startswith(' ')]),
+            'estimated_tokens': len(diff) // 4  # Rough estimation
+        }
+
+        return stats

git_llm_tool/core/exceptions.py

@@ -0,0 +1,26 @@
+"""Custom exceptions for git-llm-tool."""
+
+
+class GitLlmError(Exception):
+    """Base exception for git-llm-tool."""
+    pass
+
+
+class ConfigError(GitLlmError):
+    """Configuration-related errors."""
+    pass
+
+
+class GitError(GitLlmError):
+    """Git operation errors."""
+    pass
+
+
+class ApiError(GitLlmError):
+    """LLM API-related errors."""
+    pass
+
+
+class JiraError(GitLlmError):
+    """Jira integration errors."""
+    pass