gitflow-analytics 1.0.3__py3-none-any.whl → 1.3.11__py3-none-any.whl

gitflow_analytics/qualitative/classifiers/llm/prompts.py

@@ -0,0 +1,373 @@
+"""Prompt templates and generation for LLM commit classification.
+
+This module manages all prompt engineering for commit classification,
+including templates, versioning, and context preparation.
+
+WHY: Centralizing prompt management allows for easy experimentation,
+A/B testing, and optimization without modifying classifier logic.
+
+DESIGN DECISIONS:
+- Version prompts for tracking and rollback capability
+- Support template variables for dynamic content
+- Separate system prompts from user prompts
+- Include few-shot examples for better accuracy
+- Make prompts provider-agnostic
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional
+
+
+class PromptVersion(Enum):
+    """Versions of prompt templates for A/B testing and evolution.
+
+    WHY: Track prompt versions to measure performance improvements
+    and enable rollback if newer versions perform worse.
+    """
+
+    V1_SIMPLE = "v1_simple"  # Original simple prompt
+    V2_STRUCTURED = "v2_structured"  # More structured with examples
+    V3_CONTEXTUAL = "v3_contextual"  # Enhanced with file context
+    V4_FEWSHOT = "v4_fewshot"  # Few-shot learning with examples
+
+
+@dataclass
+class PromptTemplate:
+    """Template for generating classification prompts.
+
+    WHY: Structured templates ensure consistent prompt formatting
+    and make it easy to swap different prompt strategies.
+    """
+
+    version: PromptVersion
+    system_prompt: str
+    user_prompt_template: str
+    few_shot_examples: Optional[list[dict[str, str]]] = None
+
+    def format(self, **kwargs) -> tuple[str, str]:
+        """Format the prompt with provided variables.
+
+        Args:
+            **kwargs: Variables to substitute in the template
+
+        Returns:
+            Tuple of (system_prompt, user_prompt)
+        """
+        user_prompt = self.user_prompt_template.format(**kwargs)
+        return self.system_prompt, user_prompt
+
+
+class PromptGenerator:
+    """Generates prompts for commit classification.
+
+    WHY: Encapsulates all prompt engineering logic, making it easy
+    to experiment with different prompt strategies and optimize
+    classification accuracy.
+    """
+
+    # Streamlined categories optimized for enterprise workflows
+    CATEGORIES = {
+        "feature": "New functionality, capabilities, enhancements, additions",
+        "bugfix": "Fixes, errors, issues, crashes, bugs, corrections",
+        "maintenance": "Configuration, chores, dependencies, cleanup, refactoring, updates",
+        "integration": "Third-party services, APIs, webhooks, external systems",
+        "content": "Text, copy, documentation, README updates, comments",
+        "media": "Video, audio, streaming, players, visual assets, images",
+        "localization": "Translations, i18n, l10n, regional adaptations",
+    }
+
+    # Prompt templates for different versions
+    TEMPLATES = {
+        PromptVersion.V1_SIMPLE: PromptTemplate(
+            version=PromptVersion.V1_SIMPLE,
+            system_prompt="You are a commit classification expert.",
+            user_prompt_template="""Classify this commit message into one of these 7 categories:
+
+{categories_desc}
+
+Commit message: "{message}"{context_info}
+
+Respond with only: CATEGORY_NAME confidence_score reasoning
+Example: feature 0.85 adds new user authentication system
+
+Response:""",
+        ),
+        PromptVersion.V2_STRUCTURED: PromptTemplate(
+            version=PromptVersion.V2_STRUCTURED,
+            system_prompt="""You are an expert at classifying git commit messages.
+Your task is to categorize commits accurately based on their content and context.
+Be precise and consistent in your classifications.""",
+            user_prompt_template="""Task: Classify the following git commit into exactly ONE category.
+
+Available Categories:
+{categories_desc}
+
+Commit Information:
+- Message: "{message}"
+{context_info}
+
+Output Format: CATEGORY confidence reasoning
+- CATEGORY: One of the 7 categories above (lowercase)
+- confidence: Float between 0.0 and 1.0
+- reasoning: Brief explanation (max 10 words)
+
+Response:""",
+        ),
+        PromptVersion.V3_CONTEXTUAL: PromptTemplate(
+            version=PromptVersion.V3_CONTEXTUAL,
+            system_prompt="""You are a specialized git commit classifier with deep understanding
+of software development patterns. Consider both the commit message and file context
+to make accurate classifications.""",
+            user_prompt_template="""Analyze this commit and classify it into the most appropriate category.
+
+Categories (choose ONE):
+{categories_desc}
+
+Commit Details:
+Message: "{message}"
+{context_info}
+
+Classification Rules:
+1. Focus on the PRIMARY purpose of the commit
+2. Consider file types and patterns for additional context
+3. If multiple categories apply, choose the most significant one
+4. Be confident in clear cases, conservative when ambiguous
+
+Format: CATEGORY confidence reasoning
+Response:""",
+        ),
+        PromptVersion.V4_FEWSHOT: PromptTemplate(
+            version=PromptVersion.V4_FEWSHOT,
+            system_prompt="""You are an expert commit classifier. Classify commits based on 
+the examples provided and return results in the exact format shown.""",
+            user_prompt_template="""Learn from these examples, then classify the new commit.
+
+Examples:
+{examples}
+
+Categories:
+{categories_desc}
+
+Now classify this commit:
+Message: "{message}"
+{context_info}
+
+Response (format: CATEGORY confidence reasoning):""",
+            few_shot_examples=[
+                {
+                    "message": "feat: add user authentication",
+                    "response": "feature 0.95 adds authentication functionality",
+                },
+                {
+                    "message": "fix: resolve login crash",
+                    "response": "bugfix 0.90 fixes crash issue",
+                },
+                {
+                    "message": "chore: update dependencies",
+                    "response": "maintenance 0.85 dependency updates",
+                },
+                {"message": "docs: update README", "response": "content 0.95 documentation update"},
+                {
+                    "message": "feat: add Spanish translations",
+                    "response": "localization 0.90 adds language support",
+                },
+            ],
+        ),
+    }
+
+    def __init__(self, version: PromptVersion = PromptVersion.V3_CONTEXTUAL):
+        """Initialize prompt generator with specified version.
+
+        Args:
+            version: Prompt template version to use
+        """
+        self.version = version
+        self.template = self.TEMPLATES[version]
+        self.domain_terms = self._get_default_domain_terms()
+
+    def _get_default_domain_terms(self) -> dict[str, list[str]]:
+        """Get default domain-specific terms for context enhancement.
+
+        WHY: Domain-specific terms help the LLM understand the context
+        better and make more accurate classifications.
+        """
+        return {
+            "media": [
+                "video",
+                "audio",
+                "streaming",
+                "player",
+                "media",
+                "content",
+                "broadcast",
+                "live",
+                "recording",
+                "episode",
+                "program",
+            ],
+            "localization": [
+                "translation",
+                "i18n",
+                "l10n",
+                "locale",
+                "language",
+                "spanish",
+                "french",
+                "german",
+                "italian",
+                "portuguese",
+                "multilingual",
+            ],
+            "integration": [
+                "api",
+                "webhook",
+                "third-party",
+                "external",
+                "service",
+                "integration",
+                "sync",
+                "import",
+                "export",
+                "connector",
+            ],
+        }
+
+    def prepare_context(
+        self, message: str, files_changed: Optional[list[str]] = None
+    ) -> dict[str, Any]:
+        """Prepare context information from commit data.
+
+        Args:
+            message: Commit message
+            files_changed: Optional list of changed files
+
+        Returns:
+            Context dictionary with relevant information
+        """
+        context = {"file_extensions": [], "file_patterns": [], "domain_indicators": []}
+
+        if files_changed:
+            # Extract file extensions
+            extensions = set()
+            for file_path in files_changed:
+                ext = Path(file_path).suffix.lower()
+                if ext:
+                    extensions.add(ext)
+            context["file_extensions"] = list(extensions)
+
+            # Look for specific file patterns
+            patterns = []
+            for file_path in files_changed:
+                file_lower = file_path.lower()
+                if any(
+                    term in file_lower for term in ["config", "settings", ".env", ".yaml", ".json"]
+                ):
+                    patterns.append("configuration")
+                elif any(term in file_lower for term in ["test", "spec", "__test__"]):
+                    patterns.append("test")
+                elif any(term in file_lower for term in ["doc", "readme", "changelog"]):
+                    patterns.append("documentation")
+                elif any(
+                    term in file_lower for term in ["video", "audio", "media", ".mp4", ".mp3"]
+                ):
+                    patterns.append("media")
+            context["file_patterns"] = list(set(patterns))
+
+        # Check for domain-specific terms in message
+        message_lower = message.lower()
+        for domain, terms in self.domain_terms.items():
+            if any(term in message_lower for term in terms):
+                context["domain_indicators"].append(domain)
+
+        return context
+
+    def generate_prompt(
+        self, message: str, files_changed: Optional[list[str]] = None, include_examples: bool = True
+    ) -> tuple[str, str]:
+        """Generate classification prompt for the given commit.
+
+        Args:
+            message: Commit message to classify
+            files_changed: Optional list of changed files
+            include_examples: Whether to include few-shot examples
+
+        Returns:
+            Tuple of (system_prompt, user_prompt)
+        """
+        # Prepare context
+        context = self.prepare_context(message, files_changed)
+
+        # Format context information
+        context_info = self._format_context(context)
+
+        # Format categories description
+        categories_desc = "\n".join([f"- {cat}: {desc}" for cat, desc in self.CATEGORIES.items()])
+
+        # Prepare examples if needed
+        examples = ""
+        if include_examples and self.template.few_shot_examples:
+            examples = self._format_examples(self.template.few_shot_examples)
+
+        # Format the prompt
+        return self.template.format(
+            message=message,
+            context_info=context_info,
+            categories_desc=categories_desc,
+            examples=examples,
+        )
+
+    def _format_context(self, context: dict[str, Any]) -> str:
+        """Format context information for inclusion in prompt.
+
+        Args:
+            context: Context dictionary
+
+        Returns:
+            Formatted context string
+        """
+        parts = []
+
+        if context.get("file_extensions"):
+            parts.append(f"File types: {', '.join(context['file_extensions'])}")
+
+        if context.get("file_patterns"):
+            parts.append(f"File patterns: {', '.join(context['file_patterns'])}")
+
+        if context.get("domain_indicators"):
+            parts.append(f"Domain indicators: {', '.join(context['domain_indicators'])}")
+
+        if parts:
+            return "\n" + "\n".join(parts)
+        return ""
+
+    def _format_examples(self, examples: list[dict[str, str]]) -> str:
+        """Format few-shot examples for inclusion in prompt.
+
+        Args:
+            examples: List of example classifications
+
+        Returns:
+            Formatted examples string
+        """
+        formatted = []
+        for i, example in enumerate(examples, 1):
+            formatted.append(f"{i}. Message: \"{example['message']}\"")
+            formatted.append(f"   Response: {example['response']}")
+        return "\n".join(formatted)
+
+    def get_version_info(self) -> dict[str, Any]:
+        """Get information about the current prompt version.
+
+        Returns:
+            Dictionary with version information
+        """
+        return {
+            "version": self.version.value,
+            "has_few_shot": bool(self.template.few_shot_examples),
+            "num_examples": (
+                len(self.template.few_shot_examples) if self.template.few_shot_examples else 0
+            ),
+            "categories": list(self.CATEGORIES.keys()),
+        }

gitflow_analytics/qualitative/classifiers/llm/response_parser.py

@@ -0,0 +1,287 @@
+"""Response parsing and validation for LLM outputs.
+
+This module handles parsing of LLM responses into structured classification
+results, including validation and error handling.
+
+WHY: LLM responses can be unpredictable. Robust parsing with fallbacks
+ensures the system remains stable even with unexpected outputs.
+
+DESIGN DECISIONS:
+- Support multiple response formats for flexibility
+- Validate categories against known categories
+- Extract confidence scores with bounds checking
+- Parse reasoning text safely
+- Provide detailed error messages for debugging
+"""
+
+import logging
+import re
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ResponseParser:
+    """Parses and validates LLM classification responses.
+
+    WHY: Centralizing response parsing logic makes it easier to handle
+    different response formats and add new parsing strategies.
+    """
+
+    def __init__(self):
+        """Initialize response parser."""
+        # Regex patterns for different response formats
+        self.patterns = {
+            "standard": re.compile(r"^(\w+)\s+([\d.]+)\s+(.*)$", re.IGNORECASE),
+            "colon_separated": re.compile(r"^(\w+):\s*([\d.]+)[,\s]+(.*)$", re.IGNORECASE),
+            "json_like": re.compile(
+                r'["\']?category["\']?\s*:\s*["\']?(\w+)["\']?.*?["\']?confidence["\']?\s*:\s*([\d.]+)',
+                re.IGNORECASE | re.DOTALL,
+            ),
+            "simple": re.compile(r"^(\w+)\s+([\d.]+)$", re.IGNORECASE),
+        }
+
+    def parse_response(
+        self, response: str, valid_categories: dict[str, str]
+    ) -> tuple[str, float, str]:
+        """Parse LLM response to extract classification components.
+
+        Args:
+            response: Raw LLM response text
+            valid_categories: Dictionary of valid category names
+
+        Returns:
+            Tuple of (category, confidence, reasoning)
+        """
+        if not response:
+            logger.warning("Empty response from LLM")
+            return self._fallback_result("Empty response")
+
+        # Clean the response
+        response = response.strip()
+
+        # Try each parsing pattern
+        for pattern_name, pattern in self.patterns.items():
+            match = pattern.match(response)
+            if match:
+                return self._process_match(match, pattern_name, valid_categories)
+
+        # Try to extract just the category if nothing else works
+        category = self._extract_category_fuzzy(response, valid_categories)
+        if category:
+            logger.debug(f"Fuzzy matched category: {category} from response: {response}")
+            return category, 0.5, "Fuzzy match from response"
+
+        # Complete fallback
+        logger.warning(f"Could not parse response: {response}")
+        return self._fallback_result(f"Parse failed: {response[:50]}")
+
+    def _process_match(
+        self, match: re.Match, pattern_name: str, valid_categories: dict[str, str]
+    ) -> tuple[str, float, str]:
+        """Process a regex match to extract classification components.
+
+        Args:
+            match: Regex match object
+            pattern_name: Name of the pattern that matched
+            valid_categories: Dictionary of valid categories
+
+        Returns:
+            Tuple of (category, confidence, reasoning)
+        """
+        groups = match.groups()
+
+        # Extract category
+        category = groups[0].lower().strip()
+
+        # Validate category
+        if category not in valid_categories:
+            # Try to find closest match
+            category = self._find_closest_category(category, valid_categories)
+            if not category:
+                return self._fallback_result(f"Invalid category: {groups[0]}")
+
+        # Extract confidence
+        confidence = 0.5  # Default
+        if len(groups) > 1:
+            try:
+                confidence = float(groups[1])
+                # Clamp to valid range
+                confidence = max(0.0, min(1.0, confidence))
+            except (ValueError, TypeError):
+                logger.debug(f"Could not parse confidence: {groups[1]}")
+
+        # Extract reasoning
+        reasoning = "No reasoning provided"
+        if len(groups) > 2 and groups[2]:
+            reasoning = groups[2].strip()
+            # Clean up reasoning
+            reasoning = self._clean_reasoning(reasoning)
+        elif pattern_name == "simple":
+            reasoning = f"Classified as {category}"
+
+        return category, confidence, reasoning
+
+    def _extract_category_fuzzy(
+        self, response: str, valid_categories: dict[str, str]
+    ) -> Optional[str]:
+        """Try to extract a category using fuzzy matching.
+
+        WHY: Sometimes LLMs include extra text or formatting that
+        breaks strict parsing but the category is still identifiable.
+
+        Args:
+            response: Response text to search
+            valid_categories: Dictionary of valid categories
+
+        Returns:
+            Matched category or None
+        """
+        response_lower = response.lower()
+
+        # Look for exact category names in the response
+        for category in valid_categories:
+            if category in response_lower:
+                # Check it's not part of another word
+                pattern = r"\b" + re.escape(category) + r"\b"
+                if re.search(pattern, response_lower):
+                    return category
+
+        # Look for category descriptions
+        for category, description in valid_categories.items():
+            # Check if key terms from description appear
+            key_terms = description.lower().split(",")[0].split()
+            if len(key_terms) > 0 and key_terms[0] in response_lower:
+                return category
+
+        return None
+
+    def _find_closest_category(
+        self, candidate: str, valid_categories: dict[str, str]
+    ) -> Optional[str]:
+        """Find the closest matching category for a candidate.
+
+        WHY: Handle minor typos or variations in category names
+        to improve robustness.
+
+        Args:
+            candidate: Candidate category name
+            valid_categories: Dictionary of valid categories
+
+        Returns:
+            Closest matching category or None
+        """
+        candidate_lower = candidate.lower()
+
+        # Check for common variations
+        variations = {
+            "bug": "bugfix",
+            "fix": "bugfix",
+            "bugs": "bugfix",
+            "feat": "feature",
+            "features": "feature",
+            "maint": "maintenance",
+            "maintain": "maintenance",
+            "chore": "maintenance",
+            "docs": "content",
+            "documentation": "content",
+            "doc": "content",
+            "i18n": "localization",
+            "l10n": "localization",
+            "translation": "localization",
+            "integrate": "integration",
+            "api": "integration",
+            "video": "media",
+            "audio": "media",
+        }
+
+        if candidate_lower in variations:
+            matched = variations[candidate_lower]
+            if matched in valid_categories:
+                return matched
+
+        # Check for partial matches
+        for category in valid_categories:
+            if candidate_lower.startswith(category[:3]):
+                return category
+            if category.startswith(candidate_lower[:3]):
+                return category
+
+        return None
+
+    def _clean_reasoning(self, reasoning: str) -> str:
+        """Clean up reasoning text.
+
+        Args:
+            reasoning: Raw reasoning text
+
+        Returns:
+            Cleaned reasoning text
+        """
+        # Remove extra whitespace
+        reasoning = " ".join(reasoning.split())
+
+        # Remove quotes if present
+        if reasoning.startswith('"') and reasoning.endswith('"'):
+            reasoning = reasoning[1:-1]
+        if reasoning.startswith("'") and reasoning.endswith("'"):
+            reasoning = reasoning[1:-1]
+
+        # Truncate if too long
+        max_length = 200
+        if len(reasoning) > max_length:
+            reasoning = reasoning[:max_length] + "..."
+
+        # Ensure it's not empty
+        if not reasoning:
+            reasoning = "No reasoning provided"
+
+        return reasoning
+
+    def _fallback_result(self, error_context: str) -> tuple[str, float, str]:
+        """Generate a fallback result when parsing fails.
+
+        Args:
+            error_context: Context about the parsing failure
+
+        Returns:
+            Tuple of (category, confidence, reasoning)
+        """
+        return "maintenance", 0.1, f"Parse error: {error_context}"
+
+    def validate_classification(
+        self, category: str, confidence: float, valid_categories: dict[str, str]
+    ) -> tuple[str, float, bool]:
+        """Validate and potentially correct a classification.
+
+        Args:
+            category: Classified category
+            confidence: Confidence score
+            valid_categories: Dictionary of valid categories
+
+        Returns:
+            Tuple of (category, confidence, is_valid)
+        """
+        is_valid = True
+
+        # Check category validity
+        if category not in valid_categories:
+            # Try to correct
+            corrected = self._find_closest_category(category, valid_categories)
+            if corrected:
+                logger.debug(f"Corrected category {category} to {corrected}")
+                category = corrected
+                confidence *= 0.8  # Reduce confidence for correction
+            else:
+                logger.warning(f"Invalid category {category}, defaulting to maintenance")
+                category = "maintenance"
+                confidence = 0.1
+                is_valid = False
+
+        # Validate confidence bounds
+        if confidence < 0 or confidence > 1:
+            logger.warning(f"Invalid confidence {confidence}, clamping to [0, 1]")
+            confidence = max(0.0, min(1.0, confidence))
+            is_valid = False
+
+        return category, confidence, is_valid