ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/analysis/semantic.py

@@ -0,0 +1,205 @@
+"""Semantic analysis using LLM to compare code behavior."""
+
+from dataclasses import dataclass
+
+from ..providers.base import Provider
+
+
+@dataclass
+class SemanticDiff:
+    """Result of semantic comparison."""
+
+    variant_id: str
+    agreement_score: float  # 0.0 to 1.0
+    behavioral_differences: list[str]
+    correctness_assessment: str  # "correct", "incorrect", "uncertain"
+    edge_cases: list[str]
+    improvements: list[str]
+    concerns: list[str]
+
+    def __post_init__(self):
+        if not self.behavioral_differences:
+            self.behavioral_differences = []
+        if not self.edge_cases:
+            self.edge_cases = []
+        if not self.improvements:
+            self.improvements = []
+        if not self.concerns:
+            self.concerns = []
+
+
+class SemanticAnalyzer:
+    """Analyzes code semantics using LLM."""
+
+    def __init__(self, provider: Provider):
+        """
+        Initialize semantic analyzer.
+
+        Args:
+            provider: LLM provider for analysis
+        """
+        self.provider = provider
+
+    async def compare(
+        self,
+        baseline: str,
+        variant: str,
+        variant_id: str,
+        user_request: str,
+    ) -> SemanticDiff:
+        """
+        Compare semantic behavior of baseline and variant.
+
+        Args:
+            baseline: Baseline code
+            variant: Variant code
+            variant_id: Identifier for variant
+            user_request: Original user request
+
+        Returns:
+            SemanticDiff with behavioral comparison
+        """
+        prompt = self._build_comparison_prompt(
+            baseline=baseline,
+            variant=variant,
+            user_request=user_request
+        )
+
+        # Get analysis from LLM
+        messages = [{"role": "user", "content": prompt}]
+        response = await self.provider.generate(messages, temperature=0.3)
+
+        # Parse response
+        analysis_text = response.get("text", "")
+        return self._parse_analysis(analysis_text, variant_id)
+
+    def _build_comparison_prompt(
+        self,
+        baseline: str,
+        variant: str,
+        user_request: str
+    ) -> str:
+        """Build prompt for semantic comparison."""
+        return f"""Compare these two code implementations for the request: "{user_request}"
+
+Baseline:
+```python
+{baseline}
+```
+
+Variant:
+```python
+{variant}
+```
+
+Analyze the following:
+
+1. **Behavioral Equivalence**: Do they produce the same behavior? Score 0-10.
+
+2. **Differences**: What are the semantic differences?
+   - Different approach?
+   - Different edge case handling?
+   - Different error handling?
+
+3. **Correctness**: Which implementation is more correct for the request?
+   - State "correct", "incorrect", or "uncertain"
+   - Explain why
+
+4. **Edge Cases**: What edge cases does each handle (or not handle)?
+
+5. **Improvements**: What does the variant do better than baseline?
+
+6. **Concerns**: Any issues, bugs, or problems in the variant?
+
+Format your response as:
+SCORE: <0-10>
+DIFFERENCES:
+- <difference 1>
+- <difference 2>
+CORRECTNESS: <correct|incorrect|uncertain>
+REASON: <explanation>
+EDGE_CASES:
+- <edge case 1>
+- <edge case 2>
+IMPROVEMENTS:
+- <improvement 1>
+- <improvement 2>
+CONCERNS:
+- <concern 1>
+- <concern 2>
+"""
+
+    def _parse_analysis(self, text: str, variant_id: str) -> SemanticDiff:
+        """
+        Parse LLM analysis response.
+
+        Args:
+            text: Analysis text from LLM
+            variant_id: Variant identifier
+
+        Returns:
+            SemanticDiff with parsed results
+        """
+        lines = text.strip().split("\n")
+
+        # Extract sections
+        score = 5.0  # Default
+        differences = []
+        correctness = "uncertain"
+        edge_cases = []
+        improvements = []
+        concerns = []
+
+        current_section = None
+
+        for line in lines:
+            line = line.strip()
+
+            if line.startswith("SCORE:"):
+                try:
+                    score_text = line.split(":", 1)[1].strip()
+                    score = float(score_text.split()[0]) / 10.0  # Normalize to 0-1
+                except (ValueError, IndexError):
+                    pass
+
+            elif line.startswith("DIFFERENCES:"):
+                current_section = "differences"
+
+            elif line.startswith("CORRECTNESS:"):
+                parts = line.split(":", 1)
+                if len(parts) > 1:
+                    correctness = parts[1].strip().lower()
+                current_section = None
+
+            elif line.startswith("EDGE_CASES:"):
+                current_section = "edge_cases"
+
+            elif line.startswith("IMPROVEMENTS:"):
+                current_section = "improvements"
+
+            elif line.startswith("CONCERNS:"):
+                current_section = "concerns"
+
+            elif line.startswith("REASON:"):
+                current_section = None
+
+            elif line.startswith("-") and current_section:
+                item = line.lstrip("- ").strip()
+                if current_section == "differences":
+                    differences.append(item)
+                elif current_section == "edge_cases":
+                    edge_cases.append(item)
+                elif current_section == "improvements":
+                    improvements.append(item)
+                elif current_section == "concerns":
+                    concerns.append(item)
+
+        return SemanticDiff(
+            variant_id=variant_id,
+            agreement_score=score,
+            behavioral_differences=differences,
+            correctness_assessment=correctness,
+            edge_cases=edge_cases,
+            improvements=improvements,
+            concerns=concerns,
+        )

ctrlcode/analysis/static.py

@@ -0,0 +1,183 @@
+"""Static analysis for code quality metrics."""
+
+import subprocess
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class StaticAnalysisResult:
+    """Result of static analysis."""
+
+    variant_id: str
+    type_errors: int = 0
+    lint_issues: int = 0
+    complexity_score: float = 0.0
+    maintainability_index: float = 0.0
+    quality_score: float = 0.0  # Overall 0-1 score
+
+
+class StaticAnalyzer:
+    """Performs static analysis on code."""
+
+    def analyze(self, code: str, variant_id: str) -> StaticAnalysisResult:
+        """
+        Run static analysis on code.
+
+        Args:
+            code: Code to analyze
+            variant_id: Identifier for variant
+
+        Returns:
+            StaticAnalysisResult with quality metrics
+        """
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmppath = Path(tmpdir)
+            code_file = tmppath / "code.py"
+            code_file.write_text(code)
+
+            # Run analyses
+            type_errors = self._run_mypy(code_file)
+            lint_issues = self._run_ruff(code_file)
+            complexity = self._calculate_complexity(code)
+
+            # Calculate overall quality score
+            quality = self._calculate_quality(
+                type_errors=type_errors,
+                lint_issues=lint_issues,
+                complexity=complexity,
+            )
+
+            return StaticAnalysisResult(
+                variant_id=variant_id,
+                type_errors=type_errors,
+                lint_issues=lint_issues,
+                complexity_score=complexity,
+                quality_score=quality,
+            )
+
+    def _run_mypy(self, file_path: Path) -> int:
+        """
+        Run mypy type checker.
+
+        Args:
+            file_path: Path to code file
+
+        Returns:
+            Number of type errors
+        """
+        try:
+            result = subprocess.run(
+                ["mypy", str(file_path), "--ignore-missing-imports"],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            # Count error lines
+            errors = 0
+            for line in result.stdout.split("\n"):
+                if "error:" in line.lower():
+                    errors += 1
+
+            return errors
+
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            return 0  # If mypy unavailable, assume no errors
+
+    def _run_ruff(self, file_path: Path) -> int:
+        """
+        Run ruff linter.
+
+        Args:
+            file_path: Path to code file
+
+        Returns:
+            Number of lint issues
+        """
+        try:
+            result = subprocess.run(
+                ["ruff", "check", str(file_path)],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            # Count issue lines
+            issues = 0
+            for line in result.stdout.split("\n"):
+                if line.strip() and not line.startswith("Found"):
+                    issues += 1
+
+            return issues
+
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            return 0  # If ruff unavailable, assume no issues
+
+    def _calculate_complexity(self, code: str) -> float:
+        """
+        Calculate code complexity.
+
+        Uses simple heuristics:
+        - Lines of code
+        - Nesting depth
+        - Number of branches
+
+        Args:
+            code: Code to analyze
+
+        Returns:
+            Complexity score (lower is better)
+        """
+        lines = [line for line in code.split("\n") if line.strip() and not line.strip().startswith("#")]
+        loc = len(lines)
+
+        # Calculate max nesting depth
+        max_depth = 0
+
+        for line in lines:
+            # Count leading spaces
+            stripped = line.lstrip()
+            if stripped:
+                indent = len(line) - len(stripped)
+                depth = indent // 4  # Assuming 4-space indents
+                max_depth = max(max_depth, depth)
+
+        # Count branches (if, for, while, try)
+        branches = sum(
+            1 for line in lines
+            if any(line.strip().startswith(kw) for kw in ["if ", "elif ", "for ", "while ", "try:", "except"])
+        )
+
+        # Complexity = weighted sum
+        complexity = (loc * 0.1) + (max_depth * 2) + (branches * 1.5)
+
+        return complexity
+
+    def _calculate_quality(
+        self,
+        type_errors: int,
+        lint_issues: int,
+        complexity: float,
+    ) -> float:
+        """
+        Calculate overall quality score.
+
+        Args:
+            type_errors: Number of type errors
+            lint_issues: Number of lint issues
+            complexity: Complexity score
+
+        Returns:
+            Quality score between 0.0 and 1.0 (higher is better)
+        """
+        # Penalties
+        type_penalty = min(type_errors * 0.1, 0.4)  # Max 40% penalty
+        lint_penalty = min(lint_issues * 0.05, 0.3)  # Max 30% penalty
+        complexity_penalty = min(complexity / 50, 0.3)  # Max 30% penalty
+
+        # Start with perfect score and subtract penalties
+        score = 1.0 - (type_penalty + lint_penalty + complexity_penalty)
+
+        return max(0.0, min(1.0, score))

ctrlcode/analysis/synthesizer.py

@@ -0,0 +1,281 @@
+"""Feedback synthesis from differential analysis."""
+
+from dataclasses import dataclass
+
+from .ast_diff import ASTDiff
+from .semantic import SemanticDiff
+from .tests import TestResult
+from .static import StaticAnalysisResult
+
+
+@dataclass
+class AnalysisResult:
+    """Complete analysis from all methods."""
+
+    structural_diffs: list[ASTDiff]
+    semantic_diffs: list[SemanticDiff]
+    test_results: list[TestResult]
+    static_results: list[StaticAnalysisResult]
+    elapsed_time: float
+
+
+@dataclass
+class Feedback:
+    """Synthesized feedback for improvement."""
+
+    improvement_prompt: str
+    quality_score: float
+    consensus: list[str]
+    discrepancies: list[str]
+    changes_made: list[str]
+
+
+class FeedbackSynthesizer:
+    """Synthesizes analysis results into actionable feedback."""
+
+    def __init__(self, weights: dict[str, float] | None = None):
+        """
+        Initialize feedback synthesizer.
+
+        Args:
+            weights: Quality scoring weights for each analysis type
+        """
+        self.weights = weights or {
+            "tests": 0.4,
+            "static": 0.3,
+            "semantic": 0.3,
+        }
+
+    def synthesize(
+        self,
+        analysis: AnalysisResult,
+        iteration: int,
+        user_request: str,
+    ) -> Feedback:
+        """
+        Generate improvement feedback from analysis.
+
+        Args:
+            analysis: Complete analysis results
+            iteration: Current iteration number
+            user_request: Original user request
+
+        Returns:
+            Feedback with improvement prompt and scores
+        """
+        # Extract consensus patterns
+        consensus = self._find_consensus(analysis)
+
+        # Extract discrepancies
+        discrepancies = self._find_discrepancies(analysis)
+
+        # Calculate quality score
+        quality_score = self._calculate_quality(analysis)
+
+        # Build improvement prompt
+        improvement_prompt = self._build_improvement_prompt(
+            consensus=consensus,
+            discrepancies=discrepancies,
+            analysis=analysis,
+            iteration=iteration,
+            user_request=user_request,
+        )
+
+        return Feedback(
+            improvement_prompt=improvement_prompt,
+            quality_score=quality_score,
+            consensus=consensus,
+            discrepancies=discrepancies,
+            changes_made=[],  # Will be populated after improvement
+        )
+
+    def _find_consensus(self, analysis: AnalysisResult) -> list[str]:
+        """
+        Find patterns agreed upon by all variants.
+
+        Args:
+            analysis: Analysis results
+
+        Returns:
+            List of consensus patterns
+        """
+        consensus = []
+
+        # Check semantic agreement
+        if analysis.semantic_diffs:
+            high_agreement = [
+                d for d in analysis.semantic_diffs
+                if d.agreement_score > 0.8
+            ]
+            if len(high_agreement) == len(analysis.semantic_diffs):
+                consensus.append("All variants show high semantic agreement")
+
+        # Check structural similarity
+        if analysis.structural_diffs:
+            similar = [
+                d for d in analysis.structural_diffs
+                if d.structural_similarity > 0.8
+            ]
+            if len(similar) == len(analysis.structural_diffs):
+                consensus.append("All variants have similar structure")
+
+        # Check test passing
+        if analysis.test_results:
+            all_passed = all(t.passed for t in analysis.test_results)
+            if all_passed:
+                consensus.append("All variants pass tests")
+
+        # Check common improvements
+        improvements: dict[str, int] = {}
+        for sem_diff in analysis.semantic_diffs:
+            for imp in sem_diff.improvements:
+                improvements[imp] = improvements.get(imp, 0) + 1
+
+        # Improvements mentioned by multiple variants
+        common_improvements = [
+            imp for imp, count in improvements.items()
+            if count >= len(analysis.semantic_diffs) // 2
+        ]
+        consensus.extend(common_improvements)
+
+        return consensus
+
+    def _find_discrepancies(self, analysis: AnalysisResult) -> list[str]:
+        """
+        Find significant differences between variants.
+
+        Args:
+            analysis: Analysis results
+
+        Returns:
+            List of discrepancy descriptions
+        """
+        discrepancies = []
+
+        # Test failures
+        if analysis.test_results:
+            failed = [t for t in analysis.test_results if not t.passed]
+            if failed:
+                discrepancies.append(
+                    f"{len(failed)}/{len(analysis.test_results)} variants failed tests"
+                )
+
+        # Semantic disagreements
+        if analysis.semantic_diffs:
+            low_agreement = [
+                d for d in analysis.semantic_diffs
+                if d.agreement_score < 0.5
+            ]
+            if low_agreement:
+                discrepancies.append(
+                    f"{len(low_agreement)} variants have low semantic agreement"
+                )
+
+            # Collect concerns
+            for sem_diff in analysis.semantic_diffs:
+                for concern in sem_diff.concerns:
+                    if concern not in discrepancies:
+                        discrepancies.append(f"Concern: {concern}")
+
+        # Structural differences
+        if analysis.structural_diffs:
+            syntax_errors = [d for d in analysis.structural_diffs if d.has_syntax_error]
+            if syntax_errors:
+                discrepancies.append(f"{len(syntax_errors)} variants have syntax errors")
+
+        # Static analysis issues
+        if analysis.static_results:
+            high_errors = [s for s in analysis.static_results if s.type_errors > 3]
+            if high_errors:
+                discrepancies.append(
+                    f"{len(high_errors)} variants have multiple type errors"
+                )
+
+            high_complexity = [s for s in analysis.static_results if s.complexity_score > 20]
+            if high_complexity:
+                discrepancies.append(
+                    f"{len(high_complexity)} variants have high complexity"
+                )
+
+        return discrepancies
+
+    def _calculate_quality(self, analysis: AnalysisResult) -> float:
+        """
+        Calculate aggregate quality score.
+
+        Args:
+            analysis: Analysis results
+
+        Returns:
+            Quality score between 0.0 and 1.0
+        """
+        score = 0.0
+
+        # Test passing rate
+        if analysis.test_results:
+            passing_rate = sum(
+                1 for t in analysis.test_results if t.passed
+            ) / len(analysis.test_results)
+            score += passing_rate * self.weights["tests"]
+
+        # Static analysis score
+        if analysis.static_results:
+            avg_static = sum(
+                s.quality_score for s in analysis.static_results
+            ) / len(analysis.static_results)
+            score += avg_static * self.weights["static"]
+
+        # Semantic coherence
+        if analysis.semantic_diffs:
+            avg_semantic = sum(
+                d.agreement_score for d in analysis.semantic_diffs
+            ) / len(analysis.semantic_diffs)
+            score += avg_semantic * self.weights["semantic"]
+
+        return min(1.0, max(0.0, score))
+
+    def _build_improvement_prompt(
+        self,
+        consensus: list[str],
+        discrepancies: list[str],
+        analysis: AnalysisResult,
+        iteration: int,
+        user_request: str,
+    ) -> str:
+        """Build prompt for generating improved version."""
+        prompt = f"""Based on fuzzing iteration {iteration}, improve the code for: "{user_request}"
+
+**Consensus (keep these patterns):**
+{chr(10).join(f"- {c}" for c in consensus) if consensus else "- No strong consensus"}
+
+**Issues to resolve:**
+{chr(10).join(f"- {d}" for d in discrepancies) if discrepancies else "- No major issues"}
+
+**Quality metrics:**
+"""
+
+        # Add test results
+        if analysis.test_results:
+            pass_rate = sum(1 for t in analysis.test_results if t.passed) / len(analysis.test_results)
+            prompt += f"\n- Test pass rate: {pass_rate:.1%}"
+
+        # Add static analysis
+        if analysis.static_results:
+            avg_quality = sum(s.quality_score for s in analysis.static_results) / len(analysis.static_results)
+            prompt += f"\n- Static analysis quality: {avg_quality:.1%}"
+
+        # Add complexity
+        if analysis.static_results:
+            avg_complexity = sum(s.complexity_score for s in analysis.static_results) / len(analysis.static_results)
+            prompt += f"\n- Average complexity: {avg_complexity:.1f}"
+
+        prompt += "\n\n**Task:**\nGenerate an improved version that:\n"
+        prompt += "1. Maintains consensus elements\n"
+        prompt += "2. Resolves identified issues\n"
+        prompt += "3. Improves quality metrics\n"
+        prompt += "4. Handles edge cases properly\n"
+        prompt += "\n**IMPORTANT:** Respond with ONLY the improved output directly. "
+        prompt += "Do NOT include explanations, meta-commentary, or descriptions of changes. "
+        prompt += "Just provide the final improved result as you would respond to the original request.\n"
+
+        return prompt