codeoptix 0.1.3__py3-none-any.whl

codeoptix/adapters/gemini_cli.py

@@ -0,0 +1,370 @@
+"""Gemini CLI adapter for CodeOptix.
+
+This adapter interfaces with Google Gemini CLI, which uses the Google Generative AI
+SDK for agent-based code generation and execution. The adapter executes the CLI via
+subprocess in non-interactive mode and parses JSON output.
+
+Note: Gemini CLI is a full CLI tool that executes code and uses tools. This adapter
+provides a simplified interface for CodeOptix's evaluation framework.
+"""
+
+import json
+import os
+import subprocess
+from typing import Any
+
+from codeoptix.adapters.base import AgentAdapter, AgentOutput
+
+
+class GeminiCLIAdapter(AgentAdapter):
+    """
+    Adapter for Google Gemini CLI.
+
+    Gemini CLI is a coding agent that uses Google's Generative AI SDK. This adapter
+    executes Gemini CLI via subprocess in non-interactive mode with JSON output
+    to capture structured responses from the agent.
+
+    The adapter uses `gemini` command in non-interactive mode with JSON output
+    to capture the agent's responses, code generation, and tool execution.
+    """
+
+    def __init__(self, config: dict[str, Any]):
+        """Initialize Gemini CLI adapter."""
+        super().__init__(config)
+
+        # Get LLM configuration
+        llm_config = config.get("llm_config", {})
+        self.api_key = (
+            llm_config.get("api_key") or os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")
+        )
+        self.model = llm_config.get("model", "gemini-2.0-flash-exp")
+
+        # Gemini CLI path (defaults to system PATH)
+        self.gemini_path = config.get("gemini_path") or self._find_gemini_path()
+
+        # Working directory for Gemini execution
+        self.working_directory = config.get("working_directory") or os.getcwd()
+
+        # Output format (json or stream-json)
+        self.output_format = config.get("output_format", "json")
+
+        # Get initial prompt if provided
+        self._current_prompt = config.get("prompt") or self._get_default_prompt()
+
+    def _find_gemini_path(self) -> str:
+        """Find Gemini CLI executable path."""
+        # First check if gemini is in PATH
+        import shutil
+
+        gemini_path = shutil.which("gemini")
+        if gemini_path:
+            return gemini_path
+
+        # Fallback: try common installation locations
+        possible_paths = [
+            os.path.expanduser("~/.local/bin/gemini"),
+            "/usr/local/bin/gemini",
+            "/opt/homebrew/bin/gemini",  # macOS Homebrew
+        ]
+
+        for path in possible_paths:
+            if os.path.exists(path) and os.access(path, os.X_OK):
+                return path
+
+        # If not found, return "gemini" and let subprocess handle the error
+        return "gemini"
+
+    def _get_default_prompt(self) -> str:
+        """Get default Gemini CLI system prompt."""
+        return """You are a helpful coding assistant. Write clean, secure, and well-tested code.
+Follow best practices:
+- Write secure code (no hardcoded secrets, validate inputs)
+- Write meaningful tests
+- Follow the user's requirements"""
+
+    def execute(self, prompt: str, context: dict[str, Any] | None = None) -> AgentOutput:
+        """
+        Execute Gemini CLI with a task prompt.
+
+        Uses `gemini` command in non-interactive mode with JSON output to capture
+        the agent's responses, code generation, and tool execution.
+        """
+        context = context or {}
+
+        # Build the full prompt with context
+        full_prompt = self._build_prompt(prompt, context)
+
+        # Execute Gemini CLI
+        try:
+            result = self._execute_gemini_cli(full_prompt)
+
+            # Parse JSON output to extract code and tests
+            code, tests = self._extract_from_json(result)
+
+            return AgentOutput(
+                code=code,
+                tests=tests,
+                traces=result.get("traces", []),
+                metadata={
+                    "model": self.model,
+                    "provider": "google",
+                    "gemini_version": result.get("gemini_version"),
+                    "stats": result.get("stats"),
+                },
+                prompt_used=self._current_prompt,
+            )
+        except Exception as e:
+            return AgentOutput(
+                code="",
+                tests=None,
+                traces=[{"type": "error", "error": str(e)}],
+                metadata={"error": str(e)},
+                prompt_used=self._current_prompt,
+            )
+
+    def _build_prompt(self, prompt: str, context: dict[str, Any]) -> str:
+        """Build the full prompt including system instructions and context."""
+        parts = []
+
+        # Add system prompt if available
+        if self._current_prompt:
+            parts.append(f"System instructions: {self._current_prompt}\n")
+
+        # Add context
+        if context:
+            if "plan" in context:
+                parts.append(f"Plan: {context['plan']}\n")
+            if "requirements" in context:
+                reqs = context["requirements"]
+                if isinstance(reqs, list):
+                    reqs = "\n".join(f"- {r}" for r in reqs)
+                parts.append(f"Requirements:\n{reqs}\n")
+            if "files" in context:
+                parts.append("Files:")
+                for file_path, content in context["files"].items():
+                    parts.append(f"\n{file_path}:\n{content}")
+
+        # Add the main task
+        parts.append(f"\nTask: {prompt}")
+
+        return "\n".join(parts)
+
+    def _execute_gemini_cli(self, prompt: str) -> dict[str, Any]:
+        """
+        Execute Gemini CLI via subprocess and parse JSON output.
+
+        Uses `gemini` command with `--json` or `--stream-json` flag to get
+        structured output in non-interactive mode.
+        """
+        # Build command arguments
+        cmd = [self.gemini_path]
+
+        # Add output format (--output-format json or --output-format stream-json)
+        output_format = "stream-json" if self.output_format == "stream-json" else "json"
+        cmd.extend(["--output-format", output_format])
+
+        # Add model if specified
+        if self.model:
+            cmd.extend(["--model", self.model])
+
+        # Set up environment
+        env = os.environ.copy()
+        if self.api_key:
+            env["GOOGLE_API_KEY"] = self.api_key
+            env["GEMINI_API_KEY"] = self.api_key
+
+        # Execute and capture output
+        try:
+            process = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                env=env,
+                cwd=self.working_directory,
+            )
+
+            stdout, stderr = process.communicate(input=prompt, timeout=300)
+
+            if process.returncode != 0:
+                raise RuntimeError(f"Gemini CLI exited with code {process.returncode}: {stderr}")
+
+            # Parse JSON output
+            return self._parse_json_output(stdout, stderr)
+
+        except subprocess.TimeoutExpired:
+            process.kill()
+            raise RuntimeError("Gemini CLI execution timed out after 300 seconds")
+        except FileNotFoundError:
+            raise RuntimeError(
+                f"Gemini CLI not found at '{self.gemini_path}'. "
+                "Please install Gemini CLI: npm install -g @google/gemini-cli"
+            )
+
+    def _parse_json_output(self, stdout: str, stderr: str) -> dict[str, Any]:
+        """
+        Parse JSON output from Gemini CLI.
+
+        Gemini CLI outputs JSON in the following formats:
+        - JSON: Single JSON object with session_id, content, stats
+        - Stream JSON: JSON Lines (JSONL) with events (INIT, MESSAGE, TOOL_USE, TOOL_RESULT, RESULT)
+        """
+        traces = []
+        content = ""
+        stats = None
+        tool_calls = []
+
+        # Try to parse as single JSON object first
+        try:
+            data = json.loads(stdout.strip())
+
+            # Standard JSON format
+            if "content" in data:
+                content = data.get("content", "")
+                stats = data.get("stats")
+                traces.append({"type": "json_response", "data": data})
+
+            return {
+                "traces": traces,
+                "content": content,
+                "stats": stats,
+                "tool_calls": tool_calls,
+                "stderr": stderr,
+            }
+        except json.JSONDecodeError:
+            # Try parsing as JSONL (stream-json format)
+            pass
+
+        # Parse as JSONL (stream-json format)
+        for line in stdout.strip().split("\n"):
+            if not line.strip():
+                continue
+
+            try:
+                event = json.loads(line)
+                event_type = event.get("type", "")
+
+                traces.append(event)
+
+                if event_type == "MESSAGE":
+                    # Accumulate message content
+                    if event.get("role") == "assistant":
+                        content += event.get("content", "")
+
+                elif event_type == "TOOL_USE":
+                    tool_calls.append(
+                        {
+                            "tool_name": event.get("tool_name"),
+                            "tool_id": event.get("tool_id"),
+                            "parameters": event.get("parameters"),
+                        }
+                    )
+
+                elif event_type == "TOOL_RESULT":
+                    # Tool execution result
+                    tool_calls.append(
+                        {
+                            "tool_id": event.get("tool_id"),
+                            "status": event.get("status"),
+                            "output": event.get("output"),
+                            "error": event.get("error"),
+                        }
+                    )
+
+                elif event_type == "RESULT":
+                    stats = event.get("stats")
+
+            except json.JSONDecodeError:
+                # Skip invalid JSON lines
+                continue
+
+        return {
+            "traces": traces,
+            "content": content,
+            "stats": stats,
+            "tool_calls": tool_calls,
+            "stderr": stderr,
+        }
+
+    def _extract_from_json(self, result: dict[str, Any]) -> tuple[str, str | None]:
+        """
+        Extract code and tests from Gemini JSON output.
+
+        Looks for:
+        - Code in markdown code blocks in message content
+        - Tool calls that create/modify files (file_write, file_edit tools)
+        - Test files in tool results
+        """
+        content = result.get("content", "")
+        tool_calls = result.get("tool_calls", [])
+        code = ""
+        tests = None
+
+        # Extract code from message content (markdown code blocks)
+        import re
+
+        code_blocks = re.findall(
+            r"```(?:python|javascript|typescript)?\n(.*?)```", content, re.DOTALL
+        )
+        if code_blocks:
+            code = code_blocks[0].strip()
+
+        # Look for file operations in tool calls
+        code_files = []
+        test_files = []
+
+        for tool_call in tool_calls:
+            tool_name = tool_call.get("tool_name", "")
+            parameters = tool_call.get("parameters", {})
+            output = tool_call.get("output", "")
+
+            # Check for file write/edit operations
+            if tool_name in ["file_write", "file_edit", "write_file"]:
+                file_path = parameters.get("path") or parameters.get("file_path", "")
+                file_content = parameters.get("content") or parameters.get("file_content", "")
+
+                if file_path.endswith((".py", ".js", ".ts")):
+                    if "test" in file_path.lower() or "test_" in file_path:
+                        test_files.append((file_path, file_content))
+                    else:
+                        code_files.append((file_path, file_content))
+
+            # Check tool results for file content
+            if output and isinstance(output, str):
+                # Try to extract file paths and content from output
+                file_matches = re.findall(
+                    r"File:\s*([^\n]+)\n```[^\n]*\n(.*?)```", output, re.DOTALL
+                )
+                for file_path, file_content in file_matches:
+                    if file_path.endswith((".py", ".js", ".ts")):
+                        if "test" in file_path.lower():
+                            test_files.append((file_path, file_content))
+                        else:
+                            code_files.append((file_path, file_content))
+
+        # Use file operations if available, otherwise use extracted code
+        if code_files:
+            # Combine all code files
+            code = "\n\n".join(f"# {path}\n{content}" for path, content in code_files)
+        elif not code:
+            # Fallback to message content
+            code = content
+
+        # Extract tests
+        if test_files:
+            tests = "\n\n".join(f"# {path}\n{content}" for path, content in test_files)
+
+        return code, tests
+
+    def get_prompt(self) -> str:
+        """Get current Gemini CLI prompt."""
+        return self._current_prompt or ""
+
+    def update_prompt(self, new_prompt: str) -> None:
+        """Update Gemini CLI prompt."""
+        self._current_prompt = new_prompt
+
+    def get_adapter_type(self) -> str:
+        """Get adapter type identifier."""
+        return "gemini-cli"

codeoptix/artifacts/__init__.py

@@ -0,0 +1,5 @@
+"""Artifact management for CodeOptix."""
+
+from codeoptix.artifacts.manager import ArtifactManager
+
+__all__ = ["ArtifactManager"]

codeoptix/artifacts/manager.py

@@ -0,0 +1,193 @@
+"""Artifact management for CodeOptix."""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import yaml
+
+
+class ArtifactManager:
+    """Manages storage and retrieval of CodeOptix artifacts."""
+
+    def __init__(self, artifacts_dir: str | Path | None = None):
+        """
+        Initialize artifact manager.
+
+        Args:
+            artifacts_dir: Directory for storing artifacts (default: .codeoptix/artifacts)
+        """
+        if artifacts_dir is None:
+            artifacts_dir = Path(".codeoptix") / "artifacts"
+
+        self.artifacts_dir = Path(artifacts_dir)
+        self.artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+    def save_results(self, results: dict[str, Any], run_id: str | None = None) -> Path:
+        """
+        Save evaluation results to JSON file.
+
+        Args:
+            results: Evaluation results dictionary
+            run_id: Optional run ID (generated if not provided)
+
+        Returns:
+            Path to saved results file
+        """
+        if run_id is None:
+            run_id = self._generate_run_id()
+
+        # Add metadata
+        results_with_metadata = {
+            "run_id": run_id,
+            "timestamp": datetime.utcnow().isoformat(),
+            "version": "0.1.0",
+            **results,
+        }
+
+        # Save to file
+        results_file = self.artifacts_dir / f"results_{run_id}.json"
+        with open(results_file, "w") as f:
+            json.dump(results_with_metadata, f, indent=2, default=str)
+
+        return results_file
+
+    def load_results(self, run_id: str) -> dict[str, Any]:
+        """
+        Load evaluation results by run ID.
+
+        Args:
+            run_id: Run ID to load
+
+        Returns:
+            Results dictionary
+        """
+        results_file = self.artifacts_dir / f"results_{run_id}.json"
+
+        if not results_file.exists():
+            raise FileNotFoundError(f"Results file not found: {results_file}")
+
+        with open(results_file) as f:
+            return json.load(f)
+
+    def save_reflection(self, reflection_content: str, run_id: str | None = None) -> Path:
+        """
+        Save reflection markdown file.
+
+        Args:
+            reflection_content: Reflection markdown content
+            run_id: Optional run ID (generated if not provided)
+
+        Returns:
+            Path to saved reflection file
+        """
+        if run_id is None:
+            run_id = self._generate_run_id()
+
+        reflection_file = self.artifacts_dir / f"reflection_{run_id}.md"
+        with open(reflection_file, "w") as f:
+            f.write(reflection_content)
+
+        return reflection_file
+
+    def load_reflection(self, run_id: str) -> str:
+        """
+        Load reflection markdown by run ID.
+
+        Args:
+            run_id: Run ID to load
+
+        Returns:
+            Reflection markdown content
+        """
+        reflection_file = self.artifacts_dir / f"reflection_{run_id}.md"
+
+        if not reflection_file.exists():
+            raise FileNotFoundError(f"Reflection file not found: {reflection_file}")
+
+        with open(reflection_file) as f:
+            return f.read()
+
+    def save_evolved_prompts(
+        self, evolved_prompts: dict[str, Any], run_id: str | None = None
+    ) -> Path:
+        """
+        Save evolved prompts to YAML file.
+
+        Args:
+            evolved_prompts: Evolved prompts dictionary
+            run_id: Optional run ID (generated if not provided)
+
+        Returns:
+            Path to saved prompts file
+        """
+        if run_id is None:
+            run_id = self._generate_run_id()
+
+        # Add metadata
+        prompts_with_metadata = {
+            "version": "0.1.0",
+            "run_id": run_id,
+            "timestamp": datetime.utcnow().isoformat(),
+            **evolved_prompts,
+        }
+
+        prompts_file = self.artifacts_dir / f"evolved_prompts_{run_id}.yaml"
+        with open(prompts_file, "w") as f:
+            yaml.dump(prompts_with_metadata, f, default_flow_style=False, sort_keys=False)
+
+        return prompts_file
+
+    def load_evolved_prompts(self, run_id: str) -> dict[str, Any]:
+        """
+        Load evolved prompts by run ID.
+
+        Args:
+            run_id: Run ID to load
+
+        Returns:
+            Evolved prompts dictionary
+        """
+        prompts_file = self.artifacts_dir / f"evolved_prompts_{run_id}.yaml"
+
+        if not prompts_file.exists():
+            raise FileNotFoundError(f"Evolved prompts file not found: {prompts_file}")
+
+        with open(prompts_file) as f:
+            return yaml.safe_load(f)
+
+    def list_runs(self) -> list[dict[str, Any]]:
+        """
+        List all evaluation runs.
+
+        Returns:
+            List of run metadata dictionaries
+        """
+        runs = []
+
+        # Find all results files
+        for results_file in self.artifacts_dir.glob("results_*.json"):
+            try:
+                run_id = results_file.stem.replace("results_", "")
+                with open(results_file) as f:
+                    data = json.load(f)
+                    runs.append(
+                        {
+                            "run_id": run_id,
+                            "timestamp": data.get("timestamp"),
+                            "overall_score": data.get("overall_score"),
+                            "behaviors": list(data.get("behaviors", {}).keys()),
+                        }
+                    )
+            except Exception:
+                continue
+
+        # Sort by timestamp (newest first)
+        runs.sort(key=lambda x: x.get("timestamp", ""), reverse=True)
+        return runs
+
+    def _generate_run_id(self) -> str:
+        """Generate a unique run ID."""
+        return str(uuid4())[:8]

codeoptix/behaviors/__init__.py

@@ -0,0 +1,45 @@
+"""Behavior specifications for CodeOptix."""
+
+from codeoptix.behaviors.base import BehaviorResult, BehaviorSpec, Severity
+from codeoptix.behaviors.insecure_code import InsecureCodeBehavior
+from codeoptix.behaviors.plan_drift import PlanDriftBehavior
+from codeoptix.behaviors.vacuous_tests import VacuousTestsBehavior
+
+__all__ = [
+    "BehaviorResult",
+    "BehaviorSpec",
+    "InsecureCodeBehavior",
+    "PlanDriftBehavior",
+    "Severity",
+    "VacuousTestsBehavior",
+]
+
+# Registry of available behaviors
+BEHAVIOR_REGISTRY = {
+    "insecure-code": InsecureCodeBehavior,
+    "vacuous-tests": VacuousTestsBehavior,
+    "plan-drift": PlanDriftBehavior,
+}
+
+
+def create_behavior(name: str, config: dict | None = None) -> BehaviorSpec:
+    """
+    Factory function to create a behavior spec.
+
+    Args:
+        name: Behavior name (e.g., "insecure-code")
+        config: Optional configuration dictionary
+
+    Returns:
+        BehaviorSpec instance
+
+    Raises:
+        ValueError: If behavior name is not found
+    """
+    if name not in BEHAVIOR_REGISTRY:
+        raise ValueError(
+            f"Unknown behavior: {name}. Available behaviors: {', '.join(BEHAVIOR_REGISTRY.keys())}"
+        )
+
+    behavior_class = BEHAVIOR_REGISTRY[name]
+    return behavior_class(config=config or {})

codeoptix/behaviors/base.py

@@ -0,0 +1,81 @@
+"""Base behavior specification interface for CodeOptix."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class Severity(str, Enum):
+    """Severity levels for behavior violations."""
+
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+@dataclass
+class BehaviorResult:
+    """Result of behavior evaluation."""
+
+    behavior_name: str
+    passed: bool
+    score: float  # 0.0 to 1.0 (1.0 = perfect, 0.0 = failed)
+    evidence: list[str] = field(default_factory=list)
+    severity: Severity = Severity.MEDIUM
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Validate score range."""
+        if not 0.0 <= self.score <= 1.0:
+            raise ValueError(f"Score must be between 0.0 and 1.0, got {self.score}")
+
+
+class BehaviorSpec(ABC):
+    """Base interface for behavior specifications."""
+
+    def __init__(self, config: dict[str, Any] | None = None):
+        """
+        Initialize behavior spec with configuration.
+
+        Args:
+            config: Behavior-specific configuration dictionary
+        """
+        self.config = config or {}
+        self.name = self.get_name()
+        self.severity = Severity(self.config.get("severity", "medium"))
+        self.enabled = self.config.get("enabled", True)
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """Get the name identifier for this behavior."""
+
+    @abstractmethod
+    def get_description(self) -> str:
+        """Get human-readable description of behavior."""
+
+    @abstractmethod
+    def evaluate(
+        self,
+        agent_output: Any,  # AgentOutput from adapters
+        context: dict[str, Any] | None = None,
+    ) -> BehaviorResult:
+        """
+        Evaluate agent output against behavior spec.
+
+        Args:
+            agent_output: Output from agent adapter (AgentOutput)
+            context: Optional context (files, workspace, planning artifacts, etc.)
+
+        Returns:
+            BehaviorResult with evaluation results
+        """
+
+    def is_enabled(self) -> bool:
+        """Check if this behavior spec is enabled."""
+        return self.enabled
+
+    def get_severity(self) -> Severity:
+        """Get the severity level for this behavior."""
+        return self.severity