mcpbr 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

mcpbr/logging_config.py

@@ -0,0 +1,207 @@
+"""Structured logging configuration for mcpbr.
+
+Provides JSON-structured and human-readable log formatting, log file rotation,
+environment variable overrides, and contextual log fields for benchmark runs.
+"""
+
+import json
+import logging
+import logging.handlers
+import os
+from pathlib import Path
+from typing import Any
+
+
+class StructuredFormatter(logging.Formatter):
+    """JSON-structured log formatter.
+
+    Produces one JSON object per log record, including optional context fields
+    like task_id and benchmark when they are attached to the record.
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format a log record as a JSON string.
+
+        Args:
+            record: The log record to format.
+
+        Returns:
+            A JSON-encoded string representing the log entry.
+        """
+        log_data: dict[str, Any] = {
+            "timestamp": self.formatTime(record),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if hasattr(record, "task_id"):
+            log_data["task_id"] = record.task_id
+        if hasattr(record, "benchmark"):
+            log_data["benchmark"] = record.benchmark
+        if record.exc_info and record.exc_info[0] is not None:
+            log_data["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_data)
+
+
+class HumanFormatter(logging.Formatter):
+    """Human-readable log formatter.
+
+    Produces log lines in the format: [LEVEL] logger.name: message
+    """
+
+    FORMAT = "[%(levelname)s] %(name)s: %(message)s"
+
+    def __init__(self) -> None:
+        """Initialize the formatter with the human-readable format string."""
+        super().__init__(self.FORMAT)
+
+
+def setup_logging(
+    level: str = "INFO",
+    log_file: Path | None = None,
+    structured: bool = False,
+    max_bytes: int = 10_485_760,
+    backup_count: int = 5,
+    debug: bool = False,
+    quiet: bool = False,
+) -> None:
+    """Configure mcpbr logging.
+
+    Sets up the 'mcpbr' root logger with console and optional file handlers.
+    Supports structured JSON output, log rotation, and environment variable
+    overrides via MCPBR_LOG_LEVEL.
+
+    Args:
+        level: Default log level string (e.g., 'DEBUG', 'INFO', 'WARNING', 'ERROR').
+        log_file: Optional path to a log file. If provided, a rotating file handler
+            is added. Parent directories are created automatically.
+        structured: If True, use JSON-structured formatting. Otherwise use
+            human-readable formatting.
+        max_bytes: Maximum log file size in bytes before rotation (default 10 MB).
+        backup_count: Number of rotated backup files to keep (default 5).
+        debug: If True, override level to DEBUG.
+        quiet: If True, override level to WARNING (suppresses INFO and below).
+    """
+    mcpbr_logger = logging.getLogger("mcpbr")
+
+    # Clear existing handlers to allow reconfiguration
+    mcpbr_logger.handlers.clear()
+
+    # Determine effective log level
+    # Priority: env var > debug/quiet flags > level parameter
+    env_level = os.environ.get("MCPBR_LOG_LEVEL")
+    if env_level:
+        effective_level = getattr(logging, env_level.upper(), logging.INFO)
+    elif debug:
+        effective_level = logging.DEBUG
+    elif quiet:
+        effective_level = logging.WARNING
+    else:
+        effective_level = getattr(logging, level.upper(), logging.INFO)
+
+    mcpbr_logger.setLevel(effective_level)
+
+    # Choose formatter
+    if structured:
+        formatter: logging.Formatter = StructuredFormatter()
+    else:
+        formatter = HumanFormatter()
+
+    # Console handler (always added)
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    mcpbr_logger.addHandler(console_handler)
+
+    # File handler (optional, with rotation)
+    if log_file is not None:
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+
+        file_handler = logging.handlers.RotatingFileHandler(
+            filename=str(log_path),
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+        )
+        file_handler.setFormatter(formatter)
+        mcpbr_logger.addHandler(file_handler)
+
+    # Prevent propagation to the root logger to avoid duplicate output
+    mcpbr_logger.propagate = False
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get a named mcpbr logger.
+
+    Returns a logger under the 'mcpbr' namespace. For example,
+    get_logger('evaluation') returns the logger 'mcpbr.evaluation'.
+
+    Args:
+        name: The logger name suffix (will be prefixed with 'mcpbr.').
+
+    Returns:
+        A logging.Logger instance.
+    """
+    return logging.getLogger(f"mcpbr.{name}")
+
+
+class _ContextFilter(logging.Filter):
+    """A logging filter that injects context fields into log records."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize the filter with context key-value pairs.
+
+        Args:
+            **kwargs: Arbitrary context fields to add to every log record.
+        """
+        super().__init__()
+        self.context = kwargs
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Add context fields to the log record.
+
+        Args:
+            record: The log record being processed.
+
+        Returns:
+            Always returns True (never filters out records).
+        """
+        for key, value in self.context.items():
+            setattr(record, key, value)
+        return True
+
+
+class LogContext:
+    """Add structured context fields to log records via a filter.
+
+    Use as a context manager to temporarily inject fields like task_id and
+    benchmark into all log records produced by the given logger.
+
+    Example:
+        logger = get_logger("evaluation")
+        with LogContext(logger, task_id="django-12345", benchmark="swebench"):
+            logger.info("Starting evaluation")
+            # Log record will include task_id and benchmark fields
+    """
+
+    def __init__(self, logger: logging.Logger, **kwargs: Any) -> None:
+        """Initialize the log context.
+
+        Args:
+            logger: The logger to attach context fields to.
+            **kwargs: Context fields to add (e.g., task_id, benchmark).
+        """
+        self.logger = logger
+        self.kwargs = kwargs
+        self._filter: _ContextFilter | None = None
+
+    def __enter__(self) -> "LogContext":
+        """Enter the context and attach the filter to the logger."""
+        self._filter = _ContextFilter(**self.kwargs)
+        self.logger.addFilter(self._filter)
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        """Exit the context and remove the filter from the logger."""
+        if self._filter is not None:
+            self.logger.removeFilter(self._filter)
+            self._filter = None

mcpbr/models.py

@@ -61,6 +61,72 @@ SUPPORTED_MODELS: dict[str, ModelInfo] = {
         context_window=200000,
         notes="Resolves to latest Haiku model",
     ),
+    # OpenAI models
+    "gpt-4o": ModelInfo(
+        id="gpt-4o",
+        provider="OpenAI",
+        display_name="GPT-4o",
+        context_window=128000,
+        notes="Most capable OpenAI model with vision",
+    ),
+    "gpt-4-turbo": ModelInfo(
+        id="gpt-4-turbo",
+        provider="OpenAI",
+        display_name="GPT-4 Turbo",
+        context_window=128000,
+        notes="High capability with faster inference",
+    ),
+    "gpt-4o-mini": ModelInfo(
+        id="gpt-4o-mini",
+        provider="OpenAI",
+        display_name="GPT-4o Mini",
+        context_window=128000,
+        notes="Compact and cost-effective GPT-4o variant",
+    ),
+    # Google Gemini models
+    "gemini-2.0-flash": ModelInfo(
+        id="gemini-2.0-flash",
+        provider="Google",
+        display_name="Gemini 2.0 Flash",
+        context_window=1048576,
+        notes="Latest fast Gemini model",
+    ),
+    "gemini-1.5-pro": ModelInfo(
+        id="gemini-1.5-pro",
+        provider="Google",
+        display_name="Gemini 1.5 Pro",
+        context_window=2097152,
+        notes="High-capability model with 2M token context",
+    ),
+    "gemini-1.5-flash": ModelInfo(
+        id="gemini-1.5-flash",
+        provider="Google",
+        display_name="Gemini 1.5 Flash",
+        context_window=1048576,
+        notes="Fast and cost-effective Gemini model",
+    ),
+    # Alibaba Qwen models (via DashScope)
+    "qwen-plus": ModelInfo(
+        id="qwen-plus",
+        provider="Alibaba",
+        display_name="Qwen Plus",
+        context_window=131072,
+        notes="Balanced Qwen model for general tasks",
+    ),
+    "qwen-turbo": ModelInfo(
+        id="qwen-turbo",
+        provider="Alibaba",
+        display_name="Qwen Turbo",
+        context_window=131072,
+        notes="Fast and cost-effective Qwen model",
+    ),
+    "qwen-max": ModelInfo(
+        id="qwen-max",
+        provider="Alibaba",
+        display_name="Qwen Max",
+        context_window=131072,
+        notes="Most capable Qwen model",
+    ),
 }
 
 DEFAULT_MODEL = "sonnet"

mcpbr/preflight.py

@@ -5,10 +5,11 @@ import shutil
 from dataclasses import dataclass
 from pathlib import Path
 
-import docker
 from rich.console import Console
 from rich.table import Table
 
+import docker
+
 from .config import HarnessConfig
 
 console = Console()

mcpbr/pricing.py

@@ -6,6 +6,9 @@ calculating API costs based on token usage.
 Pricing is per million tokens (MTok) and is current as of January 2026.
 Prices may change - check official provider documentation for updates:
 - Anthropic: https://www.anthropic.com/pricing
+- OpenAI: https://openai.com/pricing
+- Google: https://ai.google.dev/pricing
+- Alibaba (Qwen): https://www.alibabacloud.com/help/en/model-studio/developer-reference/billing
 """
 
 from dataclasses import dataclass
@@ -90,6 +93,75 @@ MODEL_PRICING: dict[str, ModelPricing] = {
         cache_read_price_per_mtok=0.10,
         notes="Alias for latest Haiku model",
     ),
+    # OpenAI models
+    "gpt-4o": ModelPricing(
+        model_id="gpt-4o",
+        provider="OpenAI",
+        input_price_per_mtok=2.50,
+        output_price_per_mtok=10.00,
+        notes="Most capable OpenAI model with vision",
+    ),
+    "gpt-4-turbo": ModelPricing(
+        model_id="gpt-4-turbo",
+        provider="OpenAI",
+        input_price_per_mtok=10.00,
+        output_price_per_mtok=30.00,
+        notes="High capability with faster inference",
+    ),
+    "gpt-4o-mini": ModelPricing(
+        model_id="gpt-4o-mini",
+        provider="OpenAI",
+        input_price_per_mtok=0.15,
+        output_price_per_mtok=0.60,
+        notes="Compact and cost-effective GPT-4o variant",
+    ),
+    # Google Gemini models
+    "gemini-2.0-flash": ModelPricing(
+        model_id="gemini-2.0-flash",
+        provider="Google",
+        input_price_per_mtok=0.10,
+        output_price_per_mtok=0.40,
+        notes="Latest fast Gemini model",
+    ),
+    "gemini-1.5-pro": ModelPricing(
+        model_id="gemini-1.5-pro",
+        provider="Google",
+        input_price_per_mtok=1.25,
+        output_price_per_mtok=5.00,
+        supports_prompt_caching=True,
+        cache_creation_price_per_mtok=0.3125,
+        cache_read_price_per_mtok=0.3125,
+        notes="High-capability model with 2M token context",
+    ),
+    "gemini-1.5-flash": ModelPricing(
+        model_id="gemini-1.5-flash",
+        provider="Google",
+        input_price_per_mtok=0.075,
+        output_price_per_mtok=0.30,
+        notes="Fast and cost-effective Gemini model",
+    ),
+    # Alibaba Qwen models (via DashScope)
+    "qwen-plus": ModelPricing(
+        model_id="qwen-plus",
+        provider="Alibaba",
+        input_price_per_mtok=0.40,
+        output_price_per_mtok=1.20,
+        notes="Base tier (0-256K tokens); scales to $1.20/$3.60 for 256K-1M",
+    ),
+    "qwen-turbo": ModelPricing(
+        model_id="qwen-turbo",
+        provider="Alibaba",
+        input_price_per_mtok=0.30,
+        output_price_per_mtok=0.60,
+        notes="Fast Qwen model; pricing may vary by context tier",
+    ),
+    "qwen-max": ModelPricing(
+        model_id="qwen-max",
+        provider="Alibaba",
+        input_price_per_mtok=1.20,
+        output_price_per_mtok=6.00,
+        notes="Base tier (0-32K tokens); scales to $2.40/$12.00 for 32K-128K",
+    ),
 }