coder-eval 0.8.2__py3-none-any.whl

coder_eval/path_utils.py

@@ -0,0 +1,79 @@
+"""Path utilities for run directory management."""
+
+import platform
+from datetime import datetime
+from pathlib import Path
+
+
+TASK_LOG_FILENAME = "task.log"
+
+
+def task_log_path(run_dir: Path) -> Path:
+    """Per-task log file path inside a task run directory."""
+    return run_dir / TASK_LOG_FILENAME
+
+
+def generate_run_id() -> str:
+    """Generate filesystem-safe timestamp: YYYY-MM-DD_HH-MM-SS."""
+    return datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+
+def replicate_subdir_name(replicate_index: int) -> str:
+    """Two-digit, zero-padded directory name for a replicate (``'00'``, ``'01'``, ...).
+
+    Two-digit padding caps unique replicate names at 100 (indices 0-99); if a
+    follow-up PR ever needs >=100 replicates, widen the padding here.
+    """
+    return f"{replicate_index:02d}"
+
+
+def build_task_run_dir(
+    run_dir: Path,
+    variant_id: str,
+    task_id: str,
+    replicate_index: int = 0,
+) -> Path:
+    """Build the per-task run dir: ``<run_dir>/<variant_id>/<task_id>/<NN>/``."""
+    return run_dir / variant_id / task_id / replicate_subdir_name(replicate_index)
+
+
+def format_task_log_id(variant_id: str, task_id: str, replicate_index: int = 0) -> str:
+    """Canonical ``<variant_id>/<task_id>/<NN>`` identifier used by:
+    - Orchestrator ``_log_task_id`` (console/file log tag, streaming events)
+    - Batch ``stream_label``
+    - CLI tqdm progress-bar postfix
+
+    Shape mirrors ``build_task_run_dir`` (same three segments, same NN padding
+    via ``replicate_subdir_name``) so log tags and on-disk paths stay in
+    lockstep. Callers MUST use this helper rather than hand-rolling the
+    f-string so future format changes (e.g., NN → NNN) touch exactly one
+    place.
+    """
+    return f"{variant_id}/{task_id}/{replicate_subdir_name(replicate_index)}"
+
+
+def create_latest_symlink(runs_base: Path, run_id: str) -> None:
+    """Create/update 'latest' symlink to current run.
+
+    Gracefully handles Windows where symlinks may fail.
+
+    Args:
+        runs_base: Base directory containing all runs (e.g., "runs/")
+        run_id: ID of the current run (e.g., "2025-10-09_15-30-45")
+    """
+    latest_link = runs_base / "latest"
+    # Use relative path for symlink target (just the run_id directory name)
+    # This ensures the symlink works correctly when both are in the same directory
+    target = Path(run_id)
+
+    try:
+        # Remove existing symlink/file
+        if latest_link.exists() or latest_link.is_symlink():
+            latest_link.unlink()
+
+        # Create symlink with relative path
+        latest_link.symlink_to(target, target_is_directory=True)
+    except (OSError, NotImplementedError):
+        # Windows may not support symlinks, skip gracefully
+        if platform.system() != "Windows":
+            raise

coder_eval/plugins.py

@@ -0,0 +1,81 @@
+"""Plugin discovery for bring-your-own-agent (BYOA) extensions.
+
+External packages extend coder-eval by declaring a Python entry point in the
+``coder_eval.plugins`` group whose target is a ``register(registry)`` callable::
+
+    [project.entry-points."coder_eval.plugins"]
+    my_plugin = "my_pkg.plugin:register"
+
+At startup :func:`load_plugins` scans every installed distribution for that
+group, imports each target, and calls it with the :class:`AgentRegistry` so the
+plugin can add its agent kinds. coder-eval registers its own built-in agents
+through the *same* group (the ``coder_eval`` entry point ->
+:func:`coder_eval.agents.register_builtins`), so the discovery path is exercised
+by core itself and cannot silently rot.
+
+Discovery is idempotent and re-entrancy-safe (the ``_loaded`` flag is set before
+the scan, so a plugin that imports back into coder-eval during its own
+registration does not recurse). A plugin whose ``register`` raises is logged and
+skipped — one broken plugin never aborts startup.
+"""
+
+from __future__ import annotations
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+PLUGIN_ENTRY_POINT_GROUP = "coder_eval.plugins"
+
+# coder-eval's own built-in agents register through this same entry point. A
+# failure registering it is a real breakage (empty registry), NOT a skippable
+# third-party plugin error — so it is fatal rather than logged-and-skipped.
+BUILTIN_PLUGIN_NAME = "coder_eval"
+
+_loaded = False
+
+
+def load_plugins(*, force: bool = False) -> None:
+    """Discover and run every ``coder_eval.plugins`` entry point's ``register`` hook.
+
+    Idempotent: a second call is a no-op unless ``force=True``. The ``_loaded``
+    flag is set *before* iterating so a plugin re-entering via
+    :func:`ensure_plugins_loaded` during its own import does not recurse.
+    """
+    global _loaded
+    if _loaded and not force:
+        return
+    _loaded = True
+
+    from importlib.metadata import entry_points
+
+    from coder_eval.agents.registry import AgentRegistry
+
+    for ep in entry_points(group=PLUGIN_ENTRY_POINT_GROUP):
+        try:
+            register = ep.load()
+            register(AgentRegistry)
+        except Exception:
+            # Built-in registration failing leaves the registry empty and would
+            # surface later as a misleading "No agent registered for 'claude-code'".
+            # Keep it fatal so the real (import/registration) cause fails loudly.
+            if ep.name == BUILTIN_PLUGIN_NAME:
+                # Clear the flag so a caller that catches and retries re-runs the
+                # scan instead of getting a no-op against an empty registry.
+                _loaded = False
+                raise
+            logger.exception("Failed to load coder_eval plugin %r (%s); skipping", ep.name, ep.value)
+
+
+def ensure_plugins_loaded() -> None:
+    """Run :func:`load_plugins` once if it has not already run.
+
+    Safety-net for entry paths that do not go through CLI init (direct library
+    use, tests): registry consumers call this before reading the registry so
+    registration is always populated. ``create_agent`` calls it today; the
+    config-dispatch consumers (``parse_agent_config`` and the agent-root config
+    merge) wire it in once they become registry-driven.
+    """
+    if not _loaded:
+        load_plugins()

coder_eval/pricing.py

@@ -0,0 +1,169 @@
+"""Model pricing for cost calculation.
+
+Anthropic/OpenAI built-in rates; plugins contribute additional rates via
+``register_pricing()``. Prices are per million tokens (MTok).
+Source: https://www.anthropic.com/pricing
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ModelPricing:
+    """Pricing for a single model (per million tokens)."""
+
+    input_per_mtok: float
+    output_per_mtok: float
+    cache_write_per_mtok: float  # prompt caching write
+    cache_read_per_mtok: float  # prompt caching read
+
+
+# Official Anthropic pricing as of 2025
+# Key: CLI model name (before gateway mapping)
+_PRICING: dict[str, ModelPricing] = {
+    # Claude 4.8 / 4.7 / 4.6 / 4.5 / 4 Opus
+    "claude-opus-4-8": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4-7": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4-6": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4-6-20250514": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4-5-20251101": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4-20250514": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    # Claude 4.6 / 4.5 / 4 Sonnet
+    "claude-sonnet-4-6": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    "claude-sonnet-4-6-20250514": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    "claude-sonnet-4-5-20250929": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    "claude-sonnet-4-20250514": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    # Claude 4.5 Haiku
+    "claude-haiku-4-5-20251001": ModelPricing(0.80, 4.0, 1.0, 0.08),
+    # Claude 3.7 Sonnet
+    "claude-3-7-sonnet-20250219": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    # Claude 3.5 Sonnet
+    "claude-3-5-sonnet-20241022": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    "claude-3-5-sonnet-20240620": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    # Claude 3 Opus
+    "claude-3-opus-20240229": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    # Claude 3 Sonnet
+    "claude-3-sonnet-20240229": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    # Claude 3 Haiku
+    "claude-3-haiku-20240307": ModelPricing(0.25, 1.25, 0.30, 0.03),
+    # OpenAI GPT-5 / Codex (direct or via Azure OpenAI). Released 2025-09-23.
+    # Source: https://openai.com/api/pricing (gpt-5-codex: $1.25/M in,
+    # $0.125/M cached in, $10/M out). OpenAI does not bill cache writes
+    # separately, so cache_write == input rate.
+    "gpt-5-codex": ModelPricing(1.25, 10.0, 1.25, 0.125),
+    "gpt-5": ModelPricing(1.25, 10.0, 1.25, 0.125),
+    # gpt-5.3-codex (2026-02-24): $1.75/M input, $0.175/M cached, $14/M output.
+    "gpt-5.3-codex": ModelPricing(1.75, 14.0, 1.75, 0.175),
+    # gpt-5.4 (2026-03-05): $2.50/M input, $0.25/M cached, $15/M output.
+    "gpt-5.4": ModelPricing(2.5, 15.0, 2.5, 0.25),
+    # gpt-5.5: $5/M input, $0.50/M cached, $30/M output.
+    # CAVEAT: this flat rate does NOT model gpt-5.5's long-context surcharge
+    # (2x input / 1.5x output once a session exceeds 272K input tokens), so cost
+    # for very-large-context runs reads low. Fine for typical eval tasks; revisit
+    # if benchmarking large-context Codex runs.
+    "gpt-5.5": ModelPricing(5.0, 30.0, 5.0, 0.50),
+    # Google Gemini (AntigravityAgent, via the Gemini Developer API). Per-MTok
+    # rates from ai.google.dev/gemini-api/docs/pricing (2026). Gemini bills no
+    # separate cache-WRITE fee, so cache_write == input (the agent maps
+    # cache_creation_tokens to 0, so this value is effectively unused); cache_read
+    # is the cached-input rate (~10% of input). CAVEAT: Pro's >200K-token tier is
+    # higher ($4/$18); this flat rate reads low for very-large-context runs — fine
+    # for typical eval tasks. Keyed on the bare model id in agent.model.
+    "gemini-3-pro-preview": ModelPricing(2.0, 12.0, 2.0, 0.20),
+    "gemini-3.1-pro-preview": ModelPricing(2.0, 12.0, 2.0, 0.20),
+    "gemini-3.1-pro-preview-customtools": ModelPricing(2.0, 12.0, 2.0, 0.20),
+    "gemini-3.5-flash": ModelPricing(1.5, 9.0, 1.5, 0.15),
+    "gemini-3-flash-preview": ModelPricing(1.5, 9.0, 1.5, 0.15),
+}
+
+
+# Plugin-contributed rates (e.g. coder_eval_uipath registers UiPath models).
+# Merged over the built-in table at lookup time.
+_REGISTERED_PRICING: dict[str, ModelPricing] = {}
+
+
+def _lookup_rate(key: str) -> ModelPricing | None:
+    """Resolve a (normalized) pricing key: plugin overlay first, then built-ins.
+
+    Uses ``is not None`` rather than truthiness so a later ``__bool__`` on
+    ``ModelPricing`` (or a type change) can't make a falsy-but-present rate fall
+    through to the built-in table.
+    """
+    registered = _REGISTERED_PRICING.get(key)
+    return registered if registered is not None else _PRICING.get(key)
+
+
+def register_pricing(rates: dict[str, ModelPricing]) -> None:
+    """Merge plugin model rates into the pricing table.
+
+    Re-registering an identical rate for an existing key is idempotent; a
+    *conflicting* rate raises (reproducibility — mirrors AgentRegistry's
+    anti-shadow rule, so load order can't change a model's price).
+
+    All-or-nothing: every key is validated against the existing table before
+    any is committed, so a conflict on a later key in a multi-model batch does
+    not leave earlier keys half-registered.
+    """
+    for key, rate in rates.items():
+        existing = _lookup_rate(key)
+        if existing is not None and existing != rate:
+            raise ValueError(
+                f"pricing for {key!r} already registered as {existing}; refusing to shadow with {rate}. "
+                + "A built-in or another plugin already prices this model — check plugin load order "
+                + "(two plugins must not register conflicting rates for the same model id)."
+            )
+    _REGISTERED_PRICING.update(rates)
+
+
+# Bedrock cross-region inference-profile prefixes (mirrors
+# models.routing._BEDROCK_KNOWN_PREFIXES). A Bedrock route qualifies a bare
+# alias into e.g. ``eu.anthropic.claude-opus-4-8``; the pricing table is keyed
+# on the bare alias, so we strip these back off before the lookup.
+_BEDROCK_REGION_PREFIXES: tuple[str, ...] = ("eu.", "us.", "apac.", "global.")
+
+
+def _normalize_model(model: str) -> str:
+    """Strip Bedrock region + vendor qualifiers back to the bare pricing key.
+
+    ``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``. Idempotent on
+    already-bare aliases (``claude-opus-4-8`` / ``gpt-5-codex`` pass through),
+    so it is safe to apply unconditionally for every route.
+    """
+    model = model.strip()
+    for prefix in _BEDROCK_REGION_PREFIXES:
+        if model.startswith(prefix):
+            model = model[len(prefix) :]
+            break
+    if model.startswith("anthropic."):
+        model = model[len("anthropic.") :]
+    return model
+
+
+def calculate_cost(
+    model: str,
+    uncached_input_tokens: int,
+    output_tokens: int,
+    cache_creation_tokens: int = 0,
+    cache_read_tokens: int = 0,
+) -> float | None:
+    """Calculate cost in USD for the given token usage.
+
+    ``uncached_input_tokens`` is the fresh slice billed at the input rate — pass
+    ``TokenUsage.uncached_input_tokens``, NOT ``input_tokens`` (the derived total,
+    which already includes the cache buckets and would double-count them).
+
+    The model id is normalized (Bedrock region/vendor prefixes stripped) so a
+    qualified inference-profile id like ``eu.anthropic.claude-opus-4-8`` prices
+    the same as the bare ``claude-opus-4-8``. Returns None if the (normalized)
+    model is not in the pricing table.
+    """
+    pricing = _lookup_rate(_normalize_model(model))
+    if pricing is None:
+        return None
+
+    return (
+        uncached_input_tokens * pricing.input_per_mtok
+        + output_tokens * pricing.output_per_mtok
+        + cache_creation_tokens * pricing.cache_write_per_mtok
+        + cache_read_tokens * pricing.cache_read_per_mtok
+    ) / 1_000_000