code2llm 0.3.7__py3-none-any.whl

code2flow/nlp/normalization.py

@@ -0,0 +1,122 @@
+"""Query Normalization - Steps 1a-1e.
+
+1a. Lowercase conversion
+1b. Punctuation removal
+1c. Whitespace normalization
+1d. Unicode normalization (NFKC)
+1e. Stopword removal (optional)
+"""
+
+import re
+import unicodedata
+from typing import List, Dict, Optional
+from dataclasses import dataclass, field
+
+from .config import NormalizationConfig
+
+
+@dataclass
+class NormalizationResult:
+    """Result of query normalization."""
+    original: str
+    normalized: str
+    tokens: List[str] = field(default_factory=list)
+    language: str = "en"
+    steps_applied: List[str] = field(default_factory=list)
+
+
+class QueryNormalizer:
+    """Normalize queries for consistent processing."""
+    
+    def __init__(self, config: Optional[NormalizationConfig] = None):
+        self.config = config or NormalizationConfig()
+    
+    def normalize(self, query: str, language: str = "en") -> NormalizationResult:
+        """Apply full normalization pipeline (1a-1e)."""
+        result = NormalizationResult(
+            original=query,
+            normalized=query,
+            language=language,
+        )
+        
+        # 1d. Unicode normalization (NFKC) - do first
+        if self.config.unicode_normalize:
+            result.normalized = self._unicode_normalize(result.normalized)
+            result.steps_applied.append("unicode_nfkc")
+        
+        # 1a. Lowercase conversion
+        if self.config.lowercase:
+            result.normalized = self._lowercase(result.normalized)
+            result.steps_applied.append("lowercase")
+        
+        # 1b. Punctuation removal
+        if self.config.remove_punctuation:
+            result.normalized = self._remove_punctuation(result.normalized)
+            result.steps_applied.append("remove_punctuation")
+        
+        # 1c. Whitespace normalization
+        if self.config.normalize_whitespace:
+            result.normalized = self._normalize_whitespace(result.normalized)
+            result.steps_applied.append("normalize_whitespace")
+        
+        # 1e. Stopword removal
+        if self.config.remove_stopwords:
+            result.normalized = self._remove_stopwords(result.normalized, language)
+            result.steps_applied.append("remove_stopwords")
+        
+        # Tokenize
+        result.tokens = self._tokenize(result.normalized)
+        
+        return result
+    
+    def _unicode_normalize(self, text: str) -> str:
+        """1d. Normalize Unicode to NFKC form."""
+        return unicodedata.normalize('NFKC', text)
+    
+    def _lowercase(self, text: str) -> str:
+        """1a. Convert to lowercase."""
+        return text.lower()
+    
+    def _remove_punctuation(self, text: str) -> str:
+        """1b. Remove punctuation marks."""
+        # Keep alphanumeric, whitespace, and dots (for qualified names)
+        return re.sub(r'[^\w\s\.]', ' ', text)
+    
+    def _normalize_whitespace(self, text: str) -> str:
+        """1c. Normalize whitespace (multiple spaces -> single)."""
+        return ' '.join(text.split())
+    
+    def _remove_stopwords(self, text: str, language: str) -> str:
+        """1e. Remove stopwords."""
+        stopwords = self.config.stopwords.get(language, [])
+        words = text.split()
+        filtered = [w for w in words if w not in stopwords]
+        return ' '.join(filtered)
+    
+    def _tokenize(self, text: str) -> List[str]:
+        """Split text into tokens."""
+        return text.split()
+    
+    # Individual step methods for granular control
+    def step_1a_lowercase(self, text: str) -> str:
+        """Step 1a: Convert to lowercase."""
+        return text.lower()
+    
+    def step_1b_remove_punctuation(self, text: str) -> str:
+        """Step 1b: Remove punctuation."""
+        return re.sub(r'[^\w\s\.]', ' ', text)
+    
+    def step_1c_normalize_whitespace(self, text: str) -> str:
+        """Step 1c: Normalize whitespace."""
+        return ' '.join(text.split())
+    
+    def step_1d_unicode_normalize(self, text: str) -> str:
+        """Step 1d: Unicode NFKC normalization."""
+        return unicodedata.normalize('NFKC', text)
+    
+    def step_1e_remove_stopwords(self, text: str, language: str = "en") -> str:
+        """Step 1e: Remove stopwords."""
+        stopwords = self.config.stopwords.get(language, [])
+        words = text.split()
+        filtered = [w for w in words if w not in stopwords]
+        return ' '.join(filtered)

code2flow/nlp/pipeline.py

@@ -0,0 +1,388 @@
+"""Main NLP Pipeline - Integration of all components (Steps 4a-4e).
+
+4a. Pipeline orchestration
+4b. Result aggregation
+4c. Confidence scoring
+4d. Fallback handling
+4e. Output formatting
+"""
+
+from dataclasses import dataclass, field
+from typing import List, Dict, Optional, Any
+from pathlib import Path
+
+from .config import NLPConfig, FAST_NLP_CONFIG
+from .normalization import QueryNormalizer, NormalizationResult
+from .intent_matching import IntentMatcher, IntentMatchingResult
+from .entity_resolution import EntityResolver, EntityResolutionResult, Entity
+
+
+@dataclass
+class PipelineStage:
+    """Single pipeline stage result."""
+    stage_name: str
+    success: bool
+    result: Any
+    execution_time_ms: float = 0.0
+    error: Optional[str] = None
+
+
+@dataclass
+class NLPPipelineResult:
+    """Complete NLP pipeline result (4b-4e aggregation)."""
+    # 4b. Aggregated results
+    original_query: str
+    normalized_query: NormalizationResult
+    intent_result: IntentMatchingResult
+    entity_result: EntityResolutionResult
+    
+    # 4c. Confidence scoring
+    overall_confidence: float = 0.0
+    stage_confidences: Dict[str, float] = field(default_factory=dict)
+    
+    # 4d. Fallback information
+    fallback_used: bool = False
+    fallback_reason: Optional[str] = None
+    
+    # 4e. Formatted output
+    formatted_response: Optional[str] = None
+    action_recommendation: Optional[str] = None
+    
+    # Execution metadata
+    stages: List[PipelineStage] = field(default_factory=list)
+    total_execution_time_ms: float = 0.0
+    
+    def is_successful(self) -> bool:
+        """Check if pipeline produced actionable result."""
+        return (
+            self.overall_confidence >= 0.5
+            and self.intent_result.primary_intent is not None
+            and not self.fallback_used
+        )
+    
+    def get_intent(self) -> Optional[str]:
+        """Get resolved intent."""
+        if self.intent_result.primary_intent:
+            return self.intent_result.primary_intent.intent
+        return None
+    
+    def get_entities(self) -> List[Entity]:
+        """Get resolved entities."""
+        return self.entity_result.entities
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "original_query": self.original_query,
+            "normalized": self.normalized_query.normalized,
+            "intent": self.get_intent(),
+            "intent_confidence": self.intent_result.get_confidence(),
+            "entities": [
+                {
+                    "name": e.name,
+                    "type": e.entity_type,
+                    "confidence": e.confidence,
+                }
+                for e in self.entity_result.entities[:5]
+            ],
+            "overall_confidence": self.overall_confidence,
+            "fallback_used": self.fallback_used,
+            "action": self.action_recommendation,
+        }
+
+
+class NLPPipeline:
+    """Main NLP processing pipeline (4a-4e)."""
+    
+    def __init__(self, config: Optional[NLPConfig] = None):
+        self.config = config or FAST_NLP_CONFIG
+        
+        # Initialize components
+        self.normalizer = QueryNormalizer(self.config.normalization)
+        self.intent_matcher = IntentMatcher(self.config.intent_matching)
+        self.entity_resolver = EntityResolver(self.config.entity_resolution)
+        
+        # Execution history for context
+        self.query_history: List[str] = []
+    
+    def process(self, query: str, language: str = "en") -> NLPPipelineResult:
+        """Process query through full pipeline (4a-4e)."""
+        import time
+        
+        start_time = time.time()
+        stages = []
+        
+        # 4a. Pipeline orchestration - Step 1: Normalization
+        norm_start = time.time()
+        normalized = self._step_normalize(query, language)
+        norm_time = (time.time() - norm_start) * 1000
+        
+        stages.append(PipelineStage(
+            stage_name="normalization",
+            success=True,
+            result=normalized,
+            execution_time_ms=norm_time
+        ))
+        
+        # Step 2: Intent Matching
+        intent_start = time.time()
+        intent_result = self._step_match_intent(normalized.normalized)
+        intent_time = (time.time() - intent_start) * 1000
+        
+        intent_success = intent_result.primary_intent is not None
+        stages.append(PipelineStage(
+            stage_name="intent_matching",
+            success=intent_success,
+            result=intent_result,
+            execution_time_ms=intent_time
+        ))
+        
+        # Step 3: Entity Resolution
+        entity_start = time.time()
+        
+        # Determine expected entity types from intent
+        expected_types = self._infer_entity_types(intent_result)
+        
+        entity_result = self._step_resolve_entities(
+            normalized.normalized,
+            expected_types=expected_types,
+            context=normalized.normalized
+        )
+        entity_time = (time.time() - entity_start) * 1000
+        
+        entity_success = len(entity_result.entities) > 0
+        stages.append(PipelineStage(
+            stage_name="entity_resolution",
+            success=entity_success,
+            result=entity_result,
+            execution_time_ms=entity_time
+        ))
+        
+        # 4b. Result aggregation
+        total_time = (time.time() - start_time) * 1000
+        
+        result = NLPPipelineResult(
+            original_query=query,
+            normalized_query=normalized,
+            intent_result=intent_result,
+            entity_result=entity_result,
+            stages=stages,
+            total_execution_time_ms=total_time,
+        )
+        
+        # 4c. Confidence scoring
+        result.overall_confidence = self._calculate_overall_confidence(result)
+        result.stage_confidences = {
+            "normalization": 1.0,  # Normalization is deterministic
+            "intent": intent_result.get_confidence(),
+            "entity": self._calculate_entity_confidence(entity_result),
+        }
+        
+        # 4d. Fallback handling
+        if result.overall_confidence < 0.5:
+            result = self._apply_fallback(result)
+        
+        # 4e. Output formatting
+        result.action_recommendation = self._format_action(result)
+        result.formatted_response = self._format_response(result)
+        
+        # Update history
+        self.query_history.append(query)
+        if len(self.query_history) > 10:
+            self.query_history.pop(0)
+        
+        return result
+    
+    def _step_normalize(self, query: str, language: str) -> NormalizationResult:
+        """Step 1: Query normalization."""
+        if not self.config.enable_normalization:
+            # Skip normalization, return identity
+            return NormalizationResult(
+                original=query,
+                normalized=query,
+                tokens=query.split(),
+                language=language,
+                steps_applied=["skipped"]
+            )
+        
+        return self.normalizer.normalize(query, language)
+    
+    def _step_match_intent(self, normalized_query: str) -> IntentMatchingResult:
+        """Step 2: Intent matching."""
+        if not self.config.enable_intent_matching:
+            return IntentMatchingResult(query=normalized_query)
+        
+        return self.intent_matcher.match(
+            normalized_query,
+            context=self.query_history[-3:] if self.query_history else None
+        )
+    
+    def _step_resolve_entities(
+        self,
+        normalized_query: str,
+        expected_types: Optional[List[str]] = None,
+        context: Optional[str] = None
+    ) -> EntityResolutionResult:
+        """Step 3: Entity resolution."""
+        if not self.config.enable_entity_resolution:
+            return EntityResolutionResult(query=normalized_query)
+        
+        return self.entity_resolver.resolve(
+            normalized_query,
+            context=context,
+            expected_types=expected_types
+        )
+    
+    def _infer_entity_types(
+        self,
+        intent_result: IntentMatchingResult
+    ) -> Optional[List[str]]:
+        """Infer expected entity types from matched intent."""
+        if not intent_result.primary_intent:
+            return None
+        
+        intent = intent_result.primary_intent.intent
+        
+        # Map intents to expected entity types
+        intent_to_entities = {
+            "find_function": ["function"],
+            "find_class": ["class"],
+            "analyze_flow": ["function", "class"],
+            "show_call_graph": ["function", "class", "module"],
+            "find_dependencies": ["module", "file"],
+            "explain_code": ["function", "class"],
+        }
+        
+        return intent_to_entities.get(intent)
+    
+    def _calculate_overall_confidence(self, result: NLPPipelineResult) -> float:
+        """4c. Calculate overall pipeline confidence."""
+        weights = {
+            "intent": 0.5,
+            "entity": 0.3,
+            "normalization": 0.2,
+        }
+        
+        intent_conf = result.intent_result.get_confidence()
+        entity_conf = self._calculate_entity_confidence(result.entity_result)
+        norm_conf = 1.0  # Normalization is reliable
+        
+        overall = (
+            weights["intent"] * intent_conf +
+            weights["entity"] * entity_conf +
+            weights["normalization"] * norm_conf
+        )
+        
+        return round(overall, 3)
+    
+    def _calculate_entity_confidence(self, entity_result: EntityResolutionResult) -> float:
+        """Calculate aggregate entity confidence."""
+        if not entity_result.entities:
+            return 0.0
+        
+        # Use best entity confidence
+        best = max(e.confidence for e in entity_result.entities)
+        return best
+    
+    def _apply_fallback(self, result: NLPPipelineResult) -> NLPPipelineResult:
+        """4d. Apply fallback strategies when confidence is low."""
+        result.fallback_used = True
+        
+        # Try keyword-only matching as fallback
+        if result.intent_result.get_confidence() < 0.3:
+            # Re-run with lower thresholds
+            self.intent_matcher.config.fuzzy_threshold = 0.5
+            fallback_intent = self.intent_matcher.match(
+                result.normalized_query.normalized
+            )
+            
+            if fallback_intent.get_confidence() > result.intent_result.get_confidence():
+                result.intent_result = fallback_intent
+                result.fallback_reason = "lowered_threshold"
+            
+            # Restore original threshold
+            self.intent_matcher.config.fuzzy_threshold = self.config.intent_matching.fuzzy_threshold
+        
+        # If still no intent, default to generic search
+        if result.intent_result.get_confidence() < 0.2:
+            from .intent_matching import IntentMatch
+            result.intent_result.primary_intent = IntentMatch(
+                intent="generic_search",
+                confidence=0.3,
+                matched_phrase=result.normalized_query.normalized,
+                match_type="fallback"
+            )
+            result.fallback_reason = "generic_search"
+        
+        return result
+    
+    def _format_action(self, result: NLPPipelineResult) -> Optional[str]:
+        """4e. Format action recommendation."""
+        intent = result.get_intent()
+        entities = result.get_entities()
+        
+        if not intent:
+            return "Unable to determine action. Please clarify your query."
+        
+        # Format based on intent type
+        if intent == "find_function":
+            if entities:
+                return f"Search for function: {entities[0].name}"
+            return "Search for functions"
+        
+        elif intent == "find_class":
+            if entities:
+                return f"Search for class: {entities[0].name}"
+            return "Search for classes"
+        
+        elif intent == "analyze_flow":
+            if entities:
+                return f"Analyze control flow of: {entities[0].name}"
+            return "Analyze control flow"
+        
+        elif intent == "show_call_graph":
+            return "Generate call graph visualization"
+        
+        elif intent == "generic_search":
+            return f"Search for: {result.normalized_query.normalized}"
+        
+        return f"Execute: {intent}"
+    
+    def _format_response(self, result: NLPPipelineResult) -> str:
+        """4e. Format human-readable response."""
+        lines = [
+            f"Query: {result.original_query}",
+            f"Intent: {result.get_intent() or 'unknown'} (confidence: {result.overall_confidence:.2f})",
+        ]
+        
+        if result.entity_result.entities:
+            lines.append("Entities:")
+            for e in result.entity_result.entities[:3]:
+                lines.append(f"  - {e.name} ({e.entity_type}, {e.confidence:.2f})")
+        
+        if result.fallback_used:
+            lines.append(f"[Fallback used: {result.fallback_reason}]")
+        
+        return "\n".join(lines)
+    
+    # Individual step methods for 4a-4e
+    def step_4a_orchestrate(self, query: str) -> List[PipelineStage]:
+        """Step 4a: Pipeline orchestration."""
+        return self.process(query).stages
+    
+    def step_4b_aggregate(self, stages: List[PipelineStage]) -> NLPPipelineResult:
+        """Step 4b: Result aggregation."""
+        # This is done in process() method
+        pass
+    
+    def step_4c_confidence(self, result: NLPPipelineResult) -> float:
+        """Step 4c: Confidence scoring."""
+        return self._calculate_overall_confidence(result)
+    
+    def step_4d_fallback(self, result: NLPPipelineResult) -> NLPPipelineResult:
+        """Step 4d: Fallback handling."""
+        return self._apply_fallback(result)
+    
+    def step_4e_format(self, result: NLPPipelineResult) -> str:
+        """Step 4e: Output formatting."""
+        return self._format_response(result)

code2flow/patterns/detector.py

@@ -0,0 +1,168 @@
+"""Pattern detection for behavioral analysis."""
+
+from typing import List, Dict, Set
+from collections import defaultdict
+
+from ..core.config import Config
+from ..core.models import AnalysisResult
+
+
+class PatternDetector:
+    """Detect behavioral patterns in code."""
+    
+    def __init__(self, config: Config):
+        self.config = config
+        
+    def detect_patterns(self, result: AnalysisResult) -> List[Dict]:
+        """Detect all behavioral patterns in analysis result."""
+        patterns = []
+        
+        # Detect recursive patterns
+        if self.config.detect_recursion:
+            patterns.extend(self._detect_recursion(result))
+            
+        # Detect state machines
+        if self.config.detect_state_machines:
+            patterns.extend(self._detect_state_machines(result))
+            
+        # Detect factory patterns
+        patterns.extend(self._detect_factory_pattern(result))
+        
+        # Detect singleton pattern
+        patterns.extend(self._detect_singleton(result))
+        
+        # Detect strategy pattern
+        patterns.extend(self._detect_strategy_pattern(result))
+        
+        return patterns
+        
+    def _detect_recursion(self, result: AnalysisResult) -> List[Dict]:
+        """Detect recursive function calls."""
+        patterns = []
+        
+        for func_name, func_info in result.functions.items():
+            if func_name in func_info.calls:
+                patterns.append({
+                    'type': 'recursive',
+                    'name': f'recursive_{func_name}',
+                    'function': func_name,
+                    'confidence': 1.0,
+                    'description': f'Function {func_name} calls itself recursively'
+                })
+                
+        return patterns
+        
+    def _detect_state_machines(self, result: AnalysisResult) -> List[Dict]:
+        """Detect state machine patterns in classes."""
+        patterns = []
+        
+        for class_name, class_info in result.classes.items():
+            # Look for state-related attributes and transition methods
+            has_state = False
+            transition_methods = []
+            
+            for method in class_info.get('methods', []):
+                method_lower = method.lower()
+                if 'state' in method_lower:
+                    has_state = True
+                if any(word in method_lower for word in ['transition', 'change', 'set', 'next', 'prev']):
+                    transition_methods.append(method)
+                    
+            if has_state or transition_methods:
+                patterns.append({
+                    'type': 'state_machine',
+                    'name': f'state_machine_{class_name}',
+                    'class': class_name,
+                    'transitions': transition_methods,
+                    'confidence': 0.8 if transition_methods else 0.5,
+                    'description': f'Class {class_name} appears to implement a state machine'
+                })
+                
+        return patterns
+        
+    def _detect_factory_pattern(self, result: AnalysisResult) -> List[Dict]:
+        """Detect factory method pattern."""
+        patterns = []
+        
+        for func_name, func_info in result.functions.items():
+            # Check if function returns instances of different classes
+            name_lower = func_name.lower()
+            if 'create' in name_lower or 'factory' in name_lower or 'build' in name_lower:
+                # Check if it returns different types
+                returns_classes = self._check_returns_classes(result, func_name)
+                if returns_classes:
+                    patterns.append({
+                        'type': 'factory',
+                        'name': f'factory_{func_name}',
+                        'function': func_name,
+                        'creates': list(returns_classes),
+                        'confidence': 0.7,
+                        'description': f'Function {func_name} appears to be a factory'
+                    })
+                    
+        return patterns
+        
+    def _detect_singleton(self, result: AnalysisResult) -> List[Dict]:
+        """Detect singleton pattern."""
+        patterns = []
+        
+        for class_name, class_info in result.classes.items():
+            methods = class_info.get('methods', [])
+            
+            # Check for getInstance or similar
+            has_get_instance = any(
+                m.lower() in ('getinstance', 'get_instance', 'instance') 
+                for m in methods
+            )
+            
+            # Check for __new__ override
+            has_new = '__new__' in methods
+            
+            if has_get_instance or has_new:
+                patterns.append({
+                    'type': 'singleton',
+                    'name': f'singleton_{class_name}',
+                    'class': class_name,
+                    'confidence': 0.75,
+                    'description': f'Class {class_name} appears to be a singleton'
+                })
+                
+        return patterns
+        
+    def _detect_strategy_pattern(self, result: AnalysisResult) -> List[Dict]:
+        """Detect strategy pattern."""
+        patterns = []
+        
+        # Look for interface-like classes with execute/run methods
+        strategy_candidates = []
+        
+        for class_name, class_info in result.classes.items():
+            methods = class_info.get('methods', [])
+            
+            # Look for execute, run, or process methods
+            if any(m in methods for m in ['execute', 'run', 'process', 'apply']):
+                strategy_candidates.append(class_name)
+                
+        # Check if these are used interchangeably
+        if len(strategy_candidates) > 1:
+            for func_name, func_info in result.functions.items():
+                calls = func_info.calls
+                called_strategies = [s for s in strategy_candidates if s in str(calls)]
+                
+                if len(called_strategies) > 1:
+                    patterns.append({
+                        'type': 'strategy',
+                        'name': f'strategy_in_{func_name}',
+                        'context': func_name,
+                        'strategies': called_strategies,
+                        'confidence': 0.7,
+                        'description': f'Function {func_name} uses strategy pattern'
+                    })
+                    break
+                    
+        return patterns
+        
+    def _check_returns_classes(self, result: AnalysisResult, func_name: str) -> Set[str]:
+        """Check what classes a function might return."""
+        # Simplified - in full implementation would analyze return statements
+        return set()  # Placeholder