codeoptix 0.1.3__py3-none-any.whl

codeoptix/linters/runner.py

@@ -0,0 +1,186 @@
+"""Linter runner that orchestrates multiple linters."""
+
+import time
+from typing import Any
+
+from codeoptix.linters.bandit_linter import BanditLinter
+from codeoptix.linters.base import BaseLinter
+from codeoptix.linters.coverage_linter import CoverageLinter
+from codeoptix.linters.flake8_linter import Flake8Linter
+from codeoptix.linters.html_accessibility_linter import HTMLAccessibilityLinter
+from codeoptix.linters.language_detector import LanguageDetector
+from codeoptix.linters.mypy_linter import MypyLinter
+from codeoptix.linters.pip_audit_linter import PipAuditLinter
+from codeoptix.linters.pylint_linter import PylintLinter
+from codeoptix.linters.ruff_linter import RuffLinter
+from codeoptix.linters.safety_linter import SafetyLinter
+
+
+class LinterRunner:
+    """Runs multiple linters and aggregates results."""
+
+    # Registry of available linters (zero new dependencies)
+    LINTER_REGISTRY = {
+        # Code Quality (fastest first)
+        "ruff": RuffLinter,  # Fastest - should be first
+        "pylint": PylintLinter,
+        "flake8": Flake8Linter,
+        "mypy": MypyLinter,  # Type checking
+        # Security
+        "bandit": BanditLinter,
+        "safety": SafetyLinter,  # Dependency vulnerabilities
+        "pip-audit": PipAuditLinter,  # Package audit
+        # Testing
+        "coverage": CoverageLinter,  # Test coverage
+        # Accessibility (custom, no dependency)
+        "html-accessibility": HTMLAccessibilityLinter,  # Custom HTML analyzer
+    }
+
+    def __init__(self, config: dict[str, Any] | None = None):
+        """Initialize linter runner."""
+        self.config = config or {}
+        self.linters: dict[str, BaseLinter] = {}
+
+        # Initialize enabled linters
+        # Default: Use all available linters (zero new dependencies)
+        enabled_linters = self.config.get("linters", list(self.LINTER_REGISTRY.keys()))
+        for linter_name in enabled_linters:
+            if linter_name in self.LINTER_REGISTRY:
+                linter_config = self.config.get("linter_config", {}).get(linter_name, {})
+                try:
+                    self.linters[linter_name] = self.LINTER_REGISTRY[linter_name](linter_config)
+                except Exception:
+                    # Skip linters that fail to initialize
+                    pass
+
+    def run_linters(
+        self,
+        path: str,
+        linter_names: list[str] | None = None,
+        files: list[str] | None = None,
+        auto_detect: bool = True,
+    ) -> dict[str, Any]:
+        """
+        Run specified linters on code.
+
+        Args:
+            path: Path to code (file or directory)
+            linter_names: List of linter names to run (None = auto-detect or all enabled)
+            files: Optional list of specific files to check
+            auto_detect: Auto-detect language and select appropriate linters
+
+        Returns:
+            Dictionary with aggregated results
+        """
+        start_time = time.time()
+
+        # Auto-detect language and linters if requested
+        if linter_names is None and auto_detect:
+            if files:
+                # Detect from file list
+                detected_linters = LanguageDetector.get_linters_for_files(files)
+                # Find existing configs
+                config_files = LanguageDetector.find_config_files(path)
+                # Prioritize linters with existing configs
+                linter_names = []
+                for linter in detected_linters:
+                    if linter in config_files or linter in self.LINTER_REGISTRY:
+                        linter_names.append(linter)
+                # If no detected linters, use all available
+                if not linter_names:
+                    linter_names = list(self.linters.keys())
+            else:
+                # Use all enabled linters
+                linter_names = list(self.linters.keys())
+        elif linter_names is None:
+            linter_names = list(self.linters.keys())
+
+        # Filter to available linters
+        available_linters = {
+            name: linter
+            for name, linter in self.linters.items()
+            if name in linter_names and linter.is_available()
+        }
+
+        if not available_linters:
+            return {
+                "success": False,
+                "errors": [
+                    "No linters available. Install linters: pip install bandit pylint flake8"
+                ],
+                "results": {},
+                "summary": {
+                    "total_issues": 0,
+                    "critical": 0,
+                    "high": 0,
+                    "medium": 0,
+                    "low": 0,
+                },
+                "execution_time": time.time() - start_time,
+            }
+
+        # Run each linter
+        results = {}
+        all_issues = []
+        all_errors = []
+
+        for linter_name, linter in available_linters.items():
+            try:
+                result = linter.run(path, files=files)
+                results[linter_name] = result.to_dict()
+                all_issues.extend(result.issues)
+                all_errors.extend(result.errors)
+            except Exception as e:
+                all_errors.append(f"{linter_name} error: {e!s}")
+                results[linter_name] = {
+                    "success": False,
+                    "issues": [],
+                    "errors": [str(e)],
+                }
+
+        # Aggregate summary
+        summary = self._aggregate_summary(all_issues)
+
+        execution_time = time.time() - start_time
+
+        return {
+            "success": len(all_issues) == 0 and len(all_errors) == 0,
+            "results": results,
+            "summary": summary,
+            "issues": [issue.to_dict() for issue in all_issues],
+            "errors": all_errors,
+            "execution_time": execution_time,
+        }
+
+    def _aggregate_summary(self, issues: list) -> dict[str, int]:
+        """Aggregate issue summary by severity."""
+        summary = {
+            "total_issues": len(issues),
+            "critical": 0,
+            "high": 0,
+            "medium": 0,
+            "low": 0,
+            "info": 0,
+        }
+
+        for issue in issues:
+            severity = (
+                issue.severity if hasattr(issue, "severity") else issue.get("severity", "low")
+            )
+            if isinstance(severity, str):
+                severity_key = severity.lower()
+            else:
+                severity_key = severity.value.lower()
+
+            if severity_key in summary:
+                summary[severity_key] += 1
+
+        return summary
+
+    def get_available_linters(self) -> list[str]:
+        """Get list of available linters."""
+        return [name for name, linter in self.linters.items() if linter.is_available()]
+
+    def get_all_linters(self) -> list[str]:
+        """Get list of all registered linters."""
+        return list(self.LINTER_REGISTRY.keys())

codeoptix/linters/safety_linter.py

@@ -0,0 +1,184 @@
+"""Safety dependency vulnerability linter integration."""
+
+import json
+import subprocess
+from pathlib import Path
+
+from codeoptix.linters.base import BaseLinter, LinterIssue, LinterResult, Severity
+
+
+class SafetyLinter(BaseLinter):
+    """Safety dependency vulnerability scanner."""
+
+    def __init__(self, config: dict | None = None):
+        """Initialize Safety linter."""
+        super().__init__(config)
+        self.name = "safety"
+
+    def is_available(self) -> bool:
+        """Check if safety is available."""
+        try:
+            subprocess.run(
+                ["safety", "--version"],
+                capture_output=True,
+                timeout=5,
+                check=False,
+            )
+            return True
+        except (FileNotFoundError, subprocess.TimeoutExpired):
+            return False
+
+    def run(self, path: str, files: list[str] | None = None) -> LinterResult:
+        """Run safety on dependencies."""
+        import time
+
+        start_time = time.time()
+
+        if not self.is_available():
+            return LinterResult(
+                linter=self.name,
+                success=False,
+                issues=[],
+                errors=["Safety not found in PATH. Install with: pip install safety"],
+                execution_time=0.0,
+            )
+
+        # Safety checks requirements files or installed packages
+        # Find requirements files
+        path_obj = Path(path)
+        requirements_files = []
+
+        if path_obj.is_file() and path_obj.name in [
+            "requirements.txt",
+            "requirements-dev.txt",
+            "pyproject.toml",
+        ]:
+            requirements_files = [str(path_obj)]
+        elif path_obj.is_dir():
+            # Look for requirements files
+            for req_file in ["requirements.txt", "requirements-dev.txt", "pyproject.toml"]:
+                req_path = path_obj / req_file
+                if req_path.exists():
+                    requirements_files.append(str(req_path))
+
+        if not requirements_files:
+            # Safety can check installed packages without requirements file
+            requirements_files = []
+
+        try:
+            # Build command
+            cmd = ["safety", "check", "--json"]
+
+            # Add requirements file if found
+            if requirements_files:
+                cmd.extend(["--file", requirements_files[0]])
+
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=60,
+                check=False,
+            )
+
+            execution_time = time.time() - start_time
+            return self.parse_output(
+                result.stdout,
+                result.stderr,
+                result.returncode,
+                execution_time,
+            )
+        except subprocess.TimeoutExpired:
+            return LinterResult(
+                linter=self.name,
+                success=False,
+                issues=[],
+                errors=["Safety execution timed out"],
+                execution_time=60.0,
+            )
+        except Exception as e:
+            return LinterResult(
+                linter=self.name,
+                success=False,
+                issues=[],
+                errors=[f"Safety error: {e!s}"],
+                execution_time=time.time() - start_time,
+            )
+
+    def parse_output(
+        self,
+        output: str,
+        stderr: str,
+        returncode: int,
+        execution_time: float,
+    ) -> LinterResult:
+        """Parse safety JSON output."""
+        issues = []
+        errors = []
+
+        if stderr:
+            errors.append(stderr)
+
+        try:
+            data = json.loads(output) if output.strip() else []
+
+            for item in data:
+                package = item.get("package", "unknown")
+                vulnerability = item.get("vulnerability", "")
+                installed_version = item.get("installed_version", "")
+                affected_versions = item.get("affected_versions", "")
+                cve = item.get("CVE", "")
+
+                # Safety vulnerabilities are always high/critical
+                severity = Severity.HIGH
+                if "critical" in vulnerability.lower() or "critical" in str(item).lower():
+                    severity = Severity.CRITICAL
+
+                message = f"{package} {installed_version}: {vulnerability}"
+                if cve:
+                    message += f" (CVE: {cve})"
+                if affected_versions:
+                    message += f" [Affected: {affected_versions}]"
+
+                issue = LinterIssue(
+                    linter=self.name,
+                    severity=severity,
+                    message=message,
+                    file="requirements.txt",  # Safety checks dependencies
+                    rule_id=cve or f"SAFETY-{package}",
+                )
+                issues.append(issue)
+
+            return LinterResult(
+                linter=self.name,
+                success=returncode == 0 or len(issues) == 0,
+                issues=issues,
+                errors=errors,
+                execution_time=execution_time,
+                raw_output=output,
+            )
+        except json.JSONDecodeError:
+            # Safety might output text format
+            if output.strip():
+                for line in output.split("\n"):
+                    if line.strip() and "is vulnerable" in line.lower():
+                        severity = Severity.HIGH
+                        if "critical" in line.lower():
+                            severity = Severity.CRITICAL
+
+                        issue = LinterIssue(
+                            linter=self.name,
+                            severity=severity,
+                            message=line.strip(),
+                            file="requirements.txt",
+                        )
+                        issues.append(issue)
+
+            return LinterResult(
+                linter=self.name,
+                success=returncode == 0 or len(issues) == 0,
+                issues=issues,
+                errors=errors,
+                execution_time=execution_time,
+                raw_output=output,
+            )

codeoptix/reflection/__init__.py

@@ -0,0 +1,6 @@
+"""Reflection engine for CodeOptiX."""
+
+from codeoptix.reflection.engine import ReflectionEngine
+from codeoptix.reflection.generator import ReflectionGenerator
+
+__all__ = ["ReflectionEngine", "ReflectionGenerator"]

codeoptix/reflection/engine.py

@@ -0,0 +1,70 @@
+"""Reflection engine for CodeOptiX."""
+
+from typing import Any
+
+from codeoptix.artifacts.manager import ArtifactManager
+from codeoptix.reflection.generator import ReflectionGenerator
+
+
+class ReflectionEngine:
+    """
+    Reflection engine that analyzes evaluation results and generates insights.
+
+    Generates human-readable reflection reports identifying root causes
+    and providing actionable recommendations.
+    """
+
+    def __init__(
+        self, artifact_manager: ArtifactManager | None = None, config: dict[str, Any] | None = None
+    ):
+        """
+        Initialize reflection engine.
+
+        Args:
+            artifact_manager: Artifact manager for loading/saving
+            config: Configuration dictionary
+        """
+        self.artifact_manager = artifact_manager or ArtifactManager()
+        self.config = config or {}
+        self.generator = ReflectionGenerator(self.config.get("generator", {}))
+
+    def reflect(
+        self, results: dict[str, Any], agent_name: str | None = None, save: bool = True
+    ) -> str:
+        """
+        Generate reflection from evaluation results.
+
+        Args:
+            results: Evaluation results dictionary
+            agent_name: Optional agent name
+            save: Whether to save reflection to file
+
+        Returns:
+            Reflection markdown content
+        """
+        # Generate reflection content
+        reflection_content = self.generator.generate(results, agent_name=agent_name)
+
+        # Save if requested
+        if save:
+            run_id = results.get("run_id")
+            self.artifact_manager.save_reflection(reflection_content, run_id=run_id)
+
+        return reflection_content
+
+    def reflect_from_run_id(self, run_id: str, agent_name: str | None = None) -> str:
+        """
+        Generate reflection from a previous evaluation run.
+
+        Args:
+            run_id: Run ID to load results from
+            agent_name: Optional agent name
+
+        Returns:
+            Reflection markdown content
+        """
+        # Load results
+        results = self.artifact_manager.load_results(run_id)
+
+        # Generate reflection
+        return self.reflect(results, agent_name=agent_name, save=True)

codeoptix/reflection/generator.py

@@ -0,0 +1,209 @@
+"""Reflection content generation for CodeOptiX."""
+
+from typing import Any
+
+
+class ReflectionGenerator:
+    """Generates human-readable reflection reports from evaluation results."""
+
+    def __init__(self, config: dict[str, Any] | None = None):
+        """Initialize reflection generator."""
+        self.config = config or {}
+
+    def generate(self, results: dict[str, Any], agent_name: str | None = None) -> str:
+        """
+        Generate reflection markdown from evaluation results.
+
+        Args:
+            results: Evaluation results dictionary
+            agent_name: Optional agent name
+
+        Returns:
+            Reflection markdown content
+        """
+        run_id = results.get("run_id", "unknown")
+        timestamp = results.get("timestamp", "unknown")
+        overall_score = results.get("overall_score", 0.0)
+        behaviors = results.get("behaviors", {})
+
+        lines = []
+        lines.append("# CodeOptiX Reflection Report\n")
+        lines.append(f"**Run ID**: {run_id}  \n")
+        lines.append(f"**Timestamp**: {timestamp}  \n")
+        if agent_name:
+            lines.append(f"**Agent**: {agent_name}  \n")
+        lines.append(f"**Overall Score**: {overall_score:.2f}/1.0  \n")
+        lines.append("\n---\n")
+
+        # Summary section
+        lines.append("## Summary\n\n")
+        total_behaviors = len(behaviors)
+        passed_behaviors = sum(1 for b in behaviors.values() if b.get("scenarios_passed", 0) > 0)
+        failed_behaviors = total_behaviors - passed_behaviors
+
+        lines.append(f"- **Total Behaviors Evaluated**: {total_behaviors}\n")
+        lines.append(f"- **Behaviors Passed**: {passed_behaviors}\n")
+        lines.append(f"- **Behaviors Failed**: {failed_behaviors}\n")
+        lines.append(f"- **Overall Score**: {overall_score:.2f}/1.0\n")
+        lines.append("\n")
+
+        # Behavior analysis section
+        lines.append("## Behavior Analysis\n\n")
+
+        for behavior_name, behavior_data in behaviors.items():
+            lines.append(f"### {behavior_name}\n\n")
+
+            score = behavior_data.get("score", 0.0)
+            scenarios_tested = behavior_data.get("scenarios_tested", 0)
+            scenarios_passed = behavior_data.get("scenarios_passed", 0)
+            scenario_results = behavior_data.get("scenario_results", [])
+            severity = (
+                scenario_results[0].get("behavior_result", {}).get("severity", "medium")
+                if scenario_results
+                else "medium"
+            )
+            evidence = behavior_data.get("evidence", [])
+
+            # Status badge
+            status = "✅ PASSED" if scenarios_passed > 0 else "❌ FAILED"
+            lines.append(f"**Status**: {status}  \n")
+            lines.append(f"**Score**: {score:.2f}/1.0  \n")
+            lines.append(f"**Severity**: {severity.upper()}  \n")
+            lines.append(f"**Scenarios Tested**: {scenarios_tested}  \n")
+            lines.append(f"**Scenarios Passed**: {scenarios_passed}  \n")
+            lines.append("\n")
+
+            # Evidence
+            if evidence:
+                lines.append("**Evidence**:\n")
+                for ev in evidence[:10]:  # Limit to 10 items
+                    lines.append(f"- {ev}\n")
+                if len(evidence) > 10:
+                    lines.append(f"- ... and {len(evidence) - 10} more issues\n")
+                lines.append("\n")
+
+            # Root cause analysis
+            root_causes = self._identify_root_causes(behavior_data)
+            if root_causes:
+                lines.append("**Root Causes**:\n")
+                for cause in root_causes:
+                    lines.append(f"- {cause}\n")
+                lines.append("\n")
+
+            # Recommendations
+            recommendations = self._generate_recommendations(behavior_name, behavior_data)
+            if recommendations:
+                lines.append("**Recommendations**:\n")
+                for rec in recommendations:
+                    lines.append(f"- {rec}\n")
+                lines.append("\n")
+
+            lines.append("---\n\n")
+
+        # Overall recommendations
+        lines.append("## Overall Recommendations\n\n")
+        overall_recs = self._generate_overall_recommendations(results)
+        for rec in overall_recs:
+            lines.append(f"- {rec}\n")
+        lines.append("\n")
+
+        # Metadata
+        lines.append("## Metadata\n\n")
+        metadata = results.get("metadata", {})
+        for key, value in metadata.items():
+            lines.append(f"- **{key}**: {value}\n")
+
+        return "".join(lines)
+
+    def _identify_root_causes(self, behavior_data: dict[str, Any]) -> list[str]:
+        """Identify root causes from behavior evaluation."""
+        causes = []
+
+        evidence = behavior_data.get("evidence", [])
+        scenario_results = behavior_data.get("scenario_results", [])
+
+        # Analyze evidence patterns
+        if any("hardcoded" in ev.lower() for ev in evidence):
+            causes.append("Missing security guidelines in agent prompt")
+
+        if any("no assertions" in ev.lower() for ev in evidence):
+            causes.append("Insufficient test quality requirements in prompt")
+
+        if any("missing" in ev.lower() for ev in evidence):
+            causes.append("Agent not following planning artifacts")
+
+        # Check evaluator results
+        for scenario_result in scenario_results:
+            eval_results = scenario_result.get("evaluator_results", {})
+
+            if eval_results.get("static_analysis", {}).get("bandit"):
+                bandit_data = eval_results["static_analysis"]["bandit"]
+                if (
+                    bandit_data
+                    and bandit_data.get("metrics", {}).get("_totals", {}).get("SEVERITY.HIGH", 0)
+                    > 0
+                ):
+                    causes.append("Static analysis detected security vulnerabilities")
+
+            if eval_results.get("test_execution", {}).get("failed_count", 0) > 0:
+                causes.append("Tests are failing or have low coverage")
+
+        return list(set(causes))  # Remove duplicates
+
+    def _generate_recommendations(
+        self, behavior_name: str, behavior_data: dict[str, Any]
+    ) -> list[str]:
+        """Generate recommendations for a specific behavior."""
+        recommendations = []
+        score = behavior_data.get("score", 0.0)
+        evidence = behavior_data.get("evidence", [])
+
+        if behavior_name == "insecure-code":
+            if score < 0.7:
+                recommendations.append("Add security checklist to agent prompt")
+                recommendations.append("Include examples of secure code patterns")
+                recommendations.append("Add explicit guidance against hardcoded secrets")
+
+        elif behavior_name == "vacuous-tests":
+            if score < 0.7:
+                recommendations.append("Emphasize test quality requirements in prompt")
+                recommendations.append("Include examples of meaningful test assertions")
+                recommendations.append("Add guidance on test coverage expectations")
+
+        elif behavior_name == "plan-drift":
+            if score < 0.7:
+                recommendations.append("Strengthen alignment with planning artifacts")
+                recommendations.append("Add explicit requirement tracking in prompt")
+                recommendations.append("Include examples of plan-following behavior")
+
+        # General recommendations based on evidence
+        if len(evidence) > 5:
+            recommendations.append(
+                "Multiple issues detected - consider comprehensive prompt revision"
+            )
+
+        return recommendations
+
+    def _generate_overall_recommendations(self, results: dict[str, Any]) -> list[str]:
+        """Generate overall recommendations."""
+        recommendations = []
+        overall_score = results.get("overall_score", 0.0)
+        behaviors = results.get("behaviors", {})
+
+        if overall_score < 0.5:
+            recommendations.append("**Critical**: Agent performance is below acceptable threshold")
+            recommendations.append("Consider major prompt revision or agent configuration changes")
+        elif overall_score < 0.7:
+            recommendations.append("**Warning**: Agent performance needs improvement")
+            recommendations.append("Review and update agent prompts based on evidence")
+        else:
+            recommendations.append("Agent performance is acceptable but can be improved")
+
+        # Behavior-specific recommendations
+        failed_behaviors = [
+            name for name, data in behaviors.items() if data.get("score", 0.0) < 0.7
+        ]
+        if failed_behaviors:
+            recommendations.append(f"Focus on improving: {', '.join(failed_behaviors)}")
+
+        return recommendations

codeoptix/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility modules for CodeOptix."""