codeoptix 0.1.0__py3-none-any.whl

codeoptix/utils/config.py

@@ -0,0 +1,91 @@
+"""Configuration management for CodeOptiX."""
+
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class LLMConfig(BaseModel):
+    """LLM configuration."""
+
+    provider: str = Field(
+        default="anthropic", description="LLM provider (anthropic, openai, google)"
+    )
+    model: str = Field(default="claude-opus-4-5-20251101", description="Model name")
+    api_key: str | None = Field(default=None, description="API key (or use environment variable)")
+    temperature: float = Field(default=1.0, description="Temperature for generation")
+    max_tokens: int | None = Field(default=None, description="Max tokens")
+
+
+class AgentConfig(BaseModel):
+    """Agent configuration."""
+
+    name: str = Field(description="Agent name")
+    adapter_type: str = Field(description="Adapter type (claude-code, codex, gemini-cli)")
+    llm_config: LLMConfig = Field(description="LLM configuration")
+    prompt: str | None = Field(default=None, description="Agent prompt/policy")
+
+
+class BehaviorConfig(BaseModel):
+    """Behavior specification configuration."""
+
+    name: str = Field(description="Behavior name")
+    enabled: bool = Field(default=True, description="Whether behavior is enabled")
+    severity: str = Field(default="medium", description="Severity level")
+    config: dict[str, Any] = Field(default_factory=dict, description="Behavior-specific config")
+
+
+class CodeOptixConfig(BaseModel):
+    """Main CodeOptix configuration."""
+
+    agent: AgentConfig = Field(description="Agent configuration")
+    behaviors: list[BehaviorConfig] = Field(
+        default_factory=list, description="Behavior specifications"
+    )
+    evaluation: dict[str, Any] = Field(default_factory=dict, description="Evaluation settings")
+    reflection: dict[str, Any] = Field(default_factory=dict, description="Reflection settings")
+    evolution: dict[str, Any] = Field(default_factory=dict, description="Evolution settings")
+    artifacts_dir: str = Field(default=".codeoptix/artifacts", description="Artifacts directory")
+
+
+def load_config(config_path: str | Path) -> CodeOptixConfig:
+    """Load configuration from YAML file."""
+    config_path = Path(config_path)
+
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+
+    with open(config_path) as f:
+        config_data = yaml.safe_load(f)
+
+    # Load API keys from environment if not provided
+    if "agent" in config_data and "llm_config" in config_data["agent"]:
+        llm_config = config_data["agent"]["llm_config"]
+        provider = llm_config.get("provider", "anthropic")
+
+        if not llm_config.get("api_key"):
+            # Try to get from environment
+            env_key_map = {
+                "anthropic": "ANTHROPIC_API_KEY",
+                "openai": "OPENAI_API_KEY",
+                "google": "GOOGLE_API_KEY",
+            }
+            env_key = env_key_map.get(provider)
+            if env_key:
+                api_key = os.getenv(env_key)
+                if api_key:
+                    llm_config["api_key"] = api_key
+
+    return CodeOptixConfig(**config_data)
+
+
+def save_config(config: CodeOptixConfig, config_path: str | Path) -> None:
+    """Save configuration to YAML file."""
+    config_path = Path(config_path)
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(config_path, "w") as f:
+        yaml.dump(config.model_dump(), f, default_flow_style=False, sort_keys=False)

codeoptix/utils/llm.py

@@ -0,0 +1,332 @@
+"""LLM client abstraction for multiple providers."""
+
+import json
+import os
+import urllib.error
+import urllib.request
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Any
+
+import anthropic
+import openai
+from google import genai
+
+
+class LLMProvider(str, Enum):
+    """Supported LLM providers."""
+
+    ANTHROPIC = "anthropic"
+    OPENAI = "openai"
+    GOOGLE = "google"
+    OLLAMA = "ollama"
+
+
+class LLMClient(ABC):
+    """Abstract base class for LLM clients."""
+
+    @abstractmethod
+    def chat_completion(
+        self,
+        messages: list[dict[str, str]],
+        model: str,
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> str:
+        """Generate a chat completion."""
+
+    @abstractmethod
+    def get_available_models(self) -> list[str]:
+        """Get list of available models for this provider."""
+
+
+class AnthropicClient(LLMClient):
+    """Anthropic Claude client."""
+
+    def __init__(self, api_key: str | None = None):
+        """Initialize Anthropic client."""
+        self.client = anthropic.Anthropic(api_key=api_key)
+
+    def chat_completion(
+        self,
+        messages: list[dict[str, str]],
+        model: str = "claude-opus-4-5-20251101",
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> str:
+        """Generate a chat completion using Anthropic."""
+        # Convert messages to Anthropic format
+        system_message = None
+        anthropic_messages = []
+
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+
+            if role == "system":
+                system_message = content
+            elif role == "user":
+                anthropic_messages.append({"role": "user", "content": content})
+            elif role == "assistant":
+                anthropic_messages.append({"role": "assistant", "content": content})
+
+        response = self.client.messages.create(
+            model=model,
+            max_tokens=max_tokens or 4096,
+            temperature=temperature,
+            system=system_message,
+            messages=anthropic_messages,
+            **kwargs,
+        )
+
+        # Extract text content from response
+        if response.content and len(response.content) > 0:
+            if hasattr(response.content[0], "text"):
+                return response.content[0].text
+            return str(response.content[0])
+        return ""
+
+    def get_available_models(self) -> list[str]:
+        """Get available Anthropic models."""
+        return [
+            "claude-opus-4-5-20251101",
+            "claude-sonnet-4-5-20251101",
+            "claude-haiku-4-5-20251101",
+            "claude-3-5-sonnet-20241022",
+            "claude-3-5-haiku-20241022",
+        ]
+
+
+class OpenAIClient(LLMClient):
+    """OpenAI GPT client."""
+
+    def __init__(self, api_key: str | None = None):
+        """Initialize OpenAI client."""
+        self.client = openai.OpenAI(api_key=api_key)
+
+    def chat_completion(
+        self,
+        messages: list[dict[str, str]],
+        model: str = "gpt-5.2",
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> str:
+        """Generate a chat completion using OpenAI."""
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, temperature=temperature, max_tokens=max_tokens, **kwargs
+        )
+
+        if response.choices and len(response.choices) > 0:
+            return response.choices[0].message.content or ""
+        return ""
+
+    def get_available_models(self) -> list[str]:
+        """Get available OpenAI models."""
+        return [
+            "gpt-5.2",
+            "gpt-4o",
+            "gpt-4o-mini",
+            "gpt-4-turbo",
+            "gpt-4",
+        ]
+
+
+class GoogleClient(LLMClient):
+    """Google Gemini client using google-genai SDK."""
+
+    def __init__(self, api_key: str | None = None):
+        """Initialize Google client."""
+        # Use the new google-genai Client API
+        self.client = genai.Client(api_key=api_key) if api_key else genai.Client()
+
+    def chat_completion(
+        self,
+        messages: list[dict[str, str]],
+        model: str = "gemini-2.0-flash-exp",
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> str:
+        """Generate a chat completion using Google Gemini."""
+        # Convert messages to the new API format
+        # The new API uses Contents format with role and parts
+        contents = []
+        system_instruction = None
+
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+
+            if role == "system":
+                system_instruction = content
+            else:
+                # Create Content object with role and parts
+                contents.append({"role": role, "parts": [{"text": content}]})
+
+        # Build the config
+        config_dict = {
+            "temperature": temperature,
+        }
+        if max_tokens:
+            config_dict["max_output_tokens"] = max_tokens
+        if system_instruction:
+            config_dict["system_instruction"] = {"parts": [{"text": system_instruction}]}
+
+        # Generate content using the new API
+        response = self.client.models.generate_content(
+            model=model,
+            contents=contents,
+            config=config_dict,
+        )
+
+        # Extract text from response
+        # The new API returns response with text attribute or candidates
+        if hasattr(response, "text") and response.text:
+            return response.text
+        if hasattr(response, "candidates") and response.candidates:
+            # Handle structured response with candidates
+            candidate = response.candidates[0]
+            if hasattr(candidate, "content") and hasattr(candidate.content, "parts"):
+                text_parts = []
+                for part in candidate.content.parts:
+                    if hasattr(part, "text") and part.text:
+                        text_parts.append(part.text)
+                if text_parts:
+                    return "".join(text_parts)
+        # Fallback to string representation
+        return str(response)
+
+    def get_available_models(self) -> list[str]:
+        """Get available Google models."""
+        return [
+            "gemini-3-pro",
+            "gemini-3-flash",
+            "gemini-2.0-flash-exp",
+            "gemini-2.5-flash",
+            "gemini-2.5-pro",
+            "gemini-1.5-pro",
+        ]
+
+
+class OllamaClient(LLMClient):
+    """Ollama local model client (http://localhost:11434)."""
+
+    def __init__(self, api_key: str | None = None):
+        """Initialize Ollama client.
+
+        api_key is unused but kept for interface compatibility.
+        """
+        base = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
+        # Normalize: strip trailing slash
+        self.base_url = base.rstrip("/")
+        # Verify connection on init (best-effort, don't fail if it's down)
+        self._verify_connection()
+
+    def _verify_connection(self) -> None:
+        """Verify Ollama connection (best-effort, non-blocking)."""
+        try:
+            req = urllib.request.Request(
+                f"{self.base_url}/api/tags",
+                headers={"Content-Type": "application/json"},
+                method="GET",
+            )
+            urllib.request.urlopen(req, timeout=2)
+        except Exception:
+            # Connection failed, but don't raise - let the actual call handle it
+            # This is just a warning check
+            pass
+
+    def chat_completion(
+        self,
+        messages: list[dict[str, str]],
+        model: str = "llama3.1",
+        temperature: float = 1.0,
+        max_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> str:
+        """Generate a chat completion using a local Ollama model."""
+        payload = {
+            "model": model,
+            "messages": messages,
+            "stream": False,
+            "options": {
+                "temperature": temperature,
+            },
+        }
+        if max_tokens:
+            payload["options"]["num_predict"] = max_tokens
+
+        data = json.dumps(payload).encode("utf-8")
+        req = urllib.request.Request(
+            f"{self.base_url}/api/chat",
+            data=data,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+
+        try:
+            # Increase timeout for large models like gpt-oss:120b which can take longer
+            with urllib.request.urlopen(req, timeout=300) as resp:  # 5 minutes for large models
+                body = resp.read().decode("utf-8")
+        except urllib.error.URLError as exc:  # pragma: no cover - network/env specific
+            # Provide helpful error message
+            default_url = "http://localhost:11434"
+            if self.base_url != default_url:
+                hint = f" (OLLAMA_BASE_URL is set to {self.base_url}, default is {default_url})"
+            else:
+                hint = " (default port is 11434)"
+            raise RuntimeError(
+                f"Failed to contact Ollama at {self.base_url}. "
+                f"Is the Ollama daemon running?{hint}\n"
+                f"  Try: ollama serve\n"
+                f"  Or set OLLAMA_BASE_URL to the correct URL if using a custom port."
+            ) from exc
+
+        try:
+            obj = json.loads(body)
+        except json.JSONDecodeError as exc:  # pragma: no cover - unexpected response
+            raise RuntimeError(f"Invalid JSON from Ollama: {body!r}") from exc
+
+        # Ollama chat API: response["message"]["content"]
+        message = obj.get("message") or {}
+        content = message.get("content")
+        if isinstance(content, str):
+            return content
+        return str(content) if content is not None else ""
+
+    def get_available_models(self) -> list[str]:
+        """Get available Ollama models via /api/tags."""
+        req = urllib.request.Request(
+            f"{self.base_url}/api/tags",
+            headers={"Content-Type": "application/json"},
+            method="GET",
+        )
+        try:
+            with urllib.request.urlopen(req) as resp:
+                body = resp.read().decode("utf-8")
+            obj = json.loads(body)
+        except Exception:  # pragma: no cover - best-effort helper
+            return []
+
+        models = []
+        for m in obj.get("models", []) or []:
+            name = m.get("name")
+            if isinstance(name, str):
+                models.append(name)
+        return models
+
+
+def create_llm_client(provider: LLMProvider, api_key: str | None = None) -> LLMClient:
+    """Factory function to create an LLM client."""
+    if provider == LLMProvider.ANTHROPIC:
+        return AnthropicClient(api_key=api_key)
+    if provider == LLMProvider.OPENAI:
+        return OpenAIClient(api_key=api_key)
+    if provider == LLMProvider.GOOGLE:
+        return GoogleClient(api_key=api_key)
+    if provider == LLMProvider.OLLAMA:
+        return OllamaClient(api_key=api_key)
+    raise ValueError(f"Unsupported provider: {provider}")

codeoptix/utils/retry.py

@@ -0,0 +1,133 @@
+"""Retry utilities for resilient API calls."""
+
+import time
+from collections.abc import Callable
+from functools import wraps
+
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+
+class RetryableError(Exception):
+    """Base exception for retryable errors."""
+
+
+class APIError(RetryableError):
+    """API-related errors that can be retried."""
+
+
+class RateLimitError(APIError):
+    """Rate limit errors."""
+
+
+class TimeoutError(APIError):
+    """Timeout errors."""
+
+
+def retry_with_backoff(
+    max_attempts: int = 3,
+    initial_wait: float = 1.0,
+    max_wait: float = 60.0,
+    exponential_base: float = 2.0,
+    retry_on: tuple[type[Exception], ...] | None = None,
+):
+    """
+    Decorator for retrying functions with exponential backoff.
+
+    Args:
+        max_attempts: Maximum number of retry attempts
+        initial_wait: Initial wait time in seconds
+        max_wait: Maximum wait time in seconds
+        exponential_base: Base for exponential backoff
+        retry_on: Tuple of exception types to retry on (default: APIError, RateLimitError)
+
+    Returns:
+        Decorated function with retry logic
+    """
+    if retry_on is None:
+        retry_on = (APIError, RateLimitError, TimeoutError)
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            wait_time = initial_wait
+
+            for attempt in range(max_attempts):
+                try:
+                    return func(*args, **kwargs)
+                except retry_on as e:
+                    last_exception = e
+                    if attempt < max_attempts - 1:
+                        # Calculate wait time with exponential backoff
+                        wait_time = min(initial_wait * (exponential_base**attempt), max_wait)
+                        time.sleep(wait_time)
+                    else:
+                        # Last attempt failed
+                        raise
+                except Exception:
+                    # Non-retryable exception
+                    raise
+
+            # Should never reach here, but just in case
+            if last_exception:
+                raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+def retry_llm_call(max_attempts: int = 3, initial_wait: float = 1.0, max_wait: float = 60.0):
+    """
+    Specialized retry decorator for LLM API calls.
+
+    Handles common LLM API errors:
+    - Rate limiting
+    - Timeouts
+    - Temporary service errors
+
+    Args:
+        max_attempts: Maximum number of retry attempts
+        initial_wait: Initial wait time in seconds
+        max_wait: Maximum wait time in seconds
+
+    Returns:
+        Decorated function with LLM-specific retry logic
+    """
+    return retry(
+        stop=stop_after_attempt(max_attempts),
+        wait=wait_exponential(multiplier=initial_wait, max=max_wait),
+        retry=retry_if_exception_type((APIError, RateLimitError, TimeoutError)),
+        reraise=True,
+    )
+
+
+def handle_api_error(error: Exception, context: str | None = None) -> str:
+    """
+    Generate user-friendly error messages from API errors.
+
+    Args:
+        error: The exception that occurred
+        context: Optional context about where the error occurred
+
+    Returns:
+        User-friendly error message
+    """
+    error_type = type(error).__name__
+    error_msg = str(error)
+
+    # Common error patterns
+    if "rate limit" in error_msg.lower() or "429" in error_msg:
+        return f"Rate limit exceeded. Please wait before retrying. {context or ''}"
+    if "timeout" in error_msg.lower() or "timed out" in error_msg.lower():
+        return f"Request timed out. The service may be slow. Try again. {context or ''}"
+    if "authentication" in error_msg.lower() or "401" in error_msg or "403" in error_msg:
+        return f"Authentication failed. Please check your API key. {context or ''}"
+    if "quota" in error_msg.lower() or "insufficient" in error_msg.lower():
+        return f"API quota exceeded. Please check your account limits. {context or ''}"
+    return f"API error ({error_type}): {error_msg}. {context or ''}"

codeoptix/vendor/__init__.py

@@ -0,0 +1,2 @@
+"""Vendored third-party code for CodeOptiX."""
+

codeoptix/vendor/bloom/README.md

@@ -0,0 +1,26 @@
+# Vendored Bloom Framework
+
+This directory contains vendored code from the Bloom evaluation framework.
+
+## Source
+
+Bloom is an open-source tool for automated behavior evaluation of LLMs. Since it's not available on PyPI, we vendor the necessary components here.
+
+## Files
+
+- `utils.py` - Core utility functions for Bloom
+- `globals.py` - Global configuration and model definitions
+- `transcript_utils.py` - Transcript handling utilities
+- `prompts/` - Prompt templates for evaluation stages
+- `orchestrators/` - Orchestration logic for conversations and simulated environments
+- `scripts/` - Evaluation stage scripts (ideation, judgment)
+- `schemas/` - JSON schemas for behaviors and conversations
+
+## Usage
+
+The vendored Bloom code is used by CodeOptiX's evaluation engine for scenario generation and behavioral evaluation. It's accessed through the `codeoptix.vendor.bloom` namespace.
+
+## Modifications
+
+All imports have been updated to use the `codeoptix.vendor.bloom` namespace instead of relative imports.
+

codeoptix/vendor/bloom/__init__.py

@@ -0,0 +1,11 @@
+"""
+Vendored Bloom evaluation framework.
+
+Bloom is an open-source tool for automated behavior evaluation of LLMs.
+Since it's not available on PyPI, we vendor the necessary components here.
+
+Original source: https://github.com/anthropics/bloom (or similar)
+"""
+
+__version__ = "0.1.0-vendored"
+

codeoptix/vendor/bloom/globals.py

@@ -0,0 +1,39 @@
+NUM_RETRIES = 30  # Global variable for the number of retries for API calls
+RETRY_BASE_DELAY = 1.0  # Base delay in seconds for exponential backoff
+RETRY_MAX_DELAY = 60.0  # Maximum delay in seconds
+
+# External transcripts directory for wandb runs
+EXTERNAL_TRANSCRIPTS_DIR = "/workspace/transcripts"  # Path relative to project root
+
+models = {
+    # Anthropic
+    "claude-sonnet-4.5": {"id": "anthropic/claude-sonnet-4-5-20250929", "org": "anthropic", "name": "Claude Sonnet 4.5"},
+    "claude-opus-4.1": {"id": "anthropic/claude-opus-4-1-20250805", "org": "anthropic", "name": "Claude Opus 4.1"},
+    "claude-sonnet-4": {"id": "anthropic/claude-sonnet-4-20250514", "org": "anthropic", "name": "Claude Sonnet 4"},
+    #"claude-sonnet-3.7": {"id": "anthropic/claude-3-7-sonnet-latest", "org": "anthropic", "name": "Claude Sonnet 3.7"},
+    #"claude-opus-4": {"id": "anthropic/claude-opus-4-20250514", "org": "anthropic", "name": "Claude Opus 4"},
+    "claude-haiku-4.5": {"id": "anthropic/claude-haiku-4-5-20251001", "org": "anthropic", "name": "Claude Haiku 4.5"},
+    # OpenAI
+    #"gpt-4.1": {"id": "openai/gpt-4.1", "org": "openai", "name": "GPT-4.1"}, #max tokens: add support
+    "gpt-5": {"id": "openai/gpt-5", "org": "openai", "name": "GPT-5"},
+    "gpt-5-mini": {"id": "openai/gpt-5-mini", "org": "openai", "name": "GPT-5 Mini"},
+    #"gpt-5-nano": {"id": "openai/gpt-5-nano", "org": "openai", "name": "GPT-5 Nano"},
+    "gpt-4o": {"id": "openai/gpt-4o", "org": "openai", "name": "GPT-4o"},
+    #"gpt-4o-mini": {"id": "openai/gpt-4o-mini", "org": "openai", "name": "GPT-4o Mini"},
+    #"o3": {"id": "openai/o3", "org": "openai", "name": "OpenAI o3"},
+    "o4-mini": {"id": "openai/o4-mini", "org": "openai", "name": "OpenAI o4-mini"},
+    #"gpt-oss-20b": {"id": "openrouter/openai/gpt-oss-20b", "org": "openrouter", "name": "GPT-OSS-20B"},
+    "gpt-oss-120b": {"id": "openrouter/openai/gpt-oss-120b", "org": "openrouter", "name": "GPT-OSS-120B"},
+    "kimi-k2": {"id": "openrouter/moonshotai/kimi-k2-0905", "org": "openrouter", "name": "Kimi K2"},
+    "kimi-k2-thinking": {"id": "openrouter/moonshotai/kimi-k2-thinking", "org": "openrouter", "name": "Kimi K2 Thinking"},
+    # Other models via OpenRouter
+    #"gemini-2.5-flash": {"id": "openrouter/google/gemini-2.5-flash", "org": "openrouter", "name": "Gemini 2.5 Flash"},
+    "gemini-2.5-pro": {"id": "openrouter/google/gemini-2.5-pro", "org": "openrouter", "name": "Gemini 2.5 Pro"},
+    "gemini-3-pro-preview": {"id": "openrouter/google/gemini-3-pro-preview", "org": "openrouter"},
+
+    #"llama-3.1-70b-instruct": {"id": "openrouter/meta-llama/llama-3-70b-instruct", "org": "openrouter", "name": "LLaMA 3.1 70B Instruct"},
+    "grok-4": {"id": "openrouter/x-ai/grok-4", "org": "openrouter", "name": "Grok 4"},
+
+    "deepseek-r1": {"id": "openrouter/deepseek/deepseek-r1", "org": "openrouter", "name": "DeepSeek R1"},
+    "deepseek-v3": {"id": "openrouter/deepseek/deepseek-chat-v3-0324", "org": "openrouter"}
+}