coder-eval 0.8.2__py3-none-any.whl

coder_eval/reports_stats.py

@@ -0,0 +1,250 @@
+"""Shared statistical + prompt-config helpers for the markdown and HTML reporters.
+
+Kept in a standalone module so ``reports_html`` can consume them without
+importing ``reports_experiment`` (which in turn imports ``reports_html``
+for its HTML-write helpers, and would otherwise form a cycle).
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+import random
+import statistics as _stats
+from pathlib import Path
+
+from coder_eval.models import EvaluationResult, ExperimentVariant, TaskExperimentSummary
+
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Statistical helpers (stdlib statistics module)
+# ---------------------------------------------------------------------------
+
+
+def mean(values: list[float]) -> float:
+    return _stats.mean(values) if values else 0.0
+
+
+def stddev(values: list[float]) -> float:
+    """Sample standard deviation (Bessel-corrected). Returns 0.0 for n < 2."""
+    return _stats.stdev(values) if len(values) >= 2 else 0.0
+
+
+def welch_t_test(a: list[float], b: list[float]) -> float | None:
+    """Two-tailed p-value using Welch's t-test via stdlib NormalDist approximation.
+
+    Uses the normal approximation for the t-distribution when df is large (>30),
+    and falls back to exact Welch-Satterthwaite otherwise. Returns None if either
+    group has fewer than 2 observations.
+    """
+    n_a, n_b = len(a), len(b)
+    if n_a < 2 or n_b < 2:
+        return None
+
+    mean_a, mean_b = _stats.mean(a), _stats.mean(b)
+    var_a = _stats.variance(a)
+    var_b = _stats.variance(b)
+
+    se_sq = var_a / n_a + var_b / n_b
+    if se_sq == 0:
+        return 1.0
+
+    t_stat = abs(mean_a - mean_b) / math.sqrt(se_sq)
+
+    # Conservative normal approximation: treat t as z-score.
+    # Overestimates p slightly for small df (heavier tails), but correct in
+    # direction and sufficient for display purposes (no incomplete-beta needed).
+    return 2.0 * _stats.NormalDist().cdf(-t_stat)
+
+
+def fmt_mean_sd(values: list[float], fmt: str = ".3f") -> str:
+    """Format mean ± stddev string. Omits ± when n < 2 (stddev undefined)."""
+    if not values:
+        return "N/A"
+    m = mean(values)
+    if len(values) < 2:
+        return f"{m:{fmt}}"
+    sd = stddev(values)
+    return f"{m:{fmt}} ± {sd:{fmt}}"
+
+
+def fmt_p(p: float | None) -> str:
+    """Format p-value for display."""
+    if p is None:
+        return "—"
+    if p < 0.001:
+        return "<0.001"
+    return f"{p:.3f}"
+
+
+# ---------------------------------------------------------------------------
+# Replicate statistics helpers (stdlib random + statistics)
+# ---------------------------------------------------------------------------
+
+
+def bootstrap_mean_ci(
+    values: list[float],
+    n_resamples: int = 1000,
+    confidence: float = 0.95,
+    seed: int = 0,
+) -> tuple[float, float, float]:
+    """Percentile-bootstrap confidence interval for the mean.
+
+    Returns (mean, ci_low, ci_high). When ``len(values) < 2``, returns
+    (values[0], values[0], values[0]) or (0, 0, 0) for empty input.
+    Uses ``random.Random(seed)`` for determinism.
+    """
+    if not values:
+        return (0.0, 0.0, 0.0)
+    m = sum(values) / len(values)
+    if len(values) < 2:
+        return (m, m, m)
+    rng = random.Random(seed)
+    n = len(values)
+    resampled_means = sorted(sum(rng.choice(values) for _ in range(n)) / n for _ in range(n_resamples))
+    alpha = (1.0 - confidence) / 2.0
+    lo = resampled_means[int(alpha * n_resamples)]
+    hi = resampled_means[int((1.0 - alpha) * n_resamples) - 1]
+    return (m, lo, hi)
+
+
+def wilson_interval(successes: int, n: int, confidence: float = 0.95) -> tuple[float, float]:
+    """Wilson score interval for a binomial proportion. Returns (low, high).
+
+    More reliable than the normal approximation at small N and near 0/1.
+    """
+    if n <= 0:
+        return (0.0, 0.0)
+    z = _stats.NormalDist().inv_cdf((1.0 + confidence) / 2.0)
+    p_hat = successes / n
+    denom = 1.0 + z * z / n
+    center = (p_hat + z * z / (2.0 * n)) / denom
+    half = (z * math.sqrt(p_hat * (1.0 - p_hat) / n + z * z / (4.0 * n * n))) / denom
+    return (max(0.0, center - half), min(1.0, center + half))
+
+
+def paired_bootstrap_diff_ci(
+    a: list[float],
+    b: list[float],
+    n_resamples: int = 1000,
+    confidence: float = 0.95,
+    seed: int = 0,
+) -> tuple[float, float, float] | None:
+    """Paired bootstrap of mean(a_i - b_i).
+
+    Returns (mean_diff, ci_low, ci_high) or None if lengths differ or < 2.
+    """
+    if len(a) != len(b) or len(a) < 2:
+        return None
+    diffs = [ai - bi for ai, bi in zip(a, b, strict=True)]
+    return bootstrap_mean_ci(diffs, n_resamples=n_resamples, confidence=confidence, seed=seed)
+
+
+def cohens_d(a: list[float], b: list[float]) -> float | None:
+    """Paired Cohen's d = mean(a_i - b_i) / stddev(a_i - b_i)."""
+    if len(a) != len(b) or len(a) < 2:
+        return None
+    diffs = [ai - bi for ai, bi in zip(a, b, strict=True)]
+    s = stddev(diffs)
+    return (sum(diffs) / len(diffs)) / s if s > 0 else None
+
+
+# ---------------------------------------------------------------------------
+# Prompt config + variant-result loaders
+# ---------------------------------------------------------------------------
+
+
+def has_final_reply(result: EvaluationResult) -> bool:
+    """True iff any iteration emitted a non-empty ResultMessage.result.
+
+    Mirrors the evalboard rendering: a "final reply" is a text answer the
+    agent produced that becomes the trailing entry in the Turn timeline.
+    """
+    for t in result.iterations:
+        if t.result_summary is not None:
+            r = t.result_summary.result
+            if isinstance(r, str) and r.strip():
+                return True
+    return False
+
+
+def visible_turn_count(result: EvaluationResult) -> int:
+    """Count of agent actions visible in the timeline so far.
+
+    A "turn" here is one entry rendered in the Turn timeline: each tool
+    invocation contributes 1, plus 1 for the final assistant reply when
+    present. This is the canonical metric — distinct from the SDK's
+    ``num_turns`` which counts assistant *messages* and can bundle tool
+    use with trailing text into a single turn.
+    """
+    commands = sum(len(t.commands) for t in result.iterations)
+    return commands + (1 if has_final_reply(result) else 0)
+
+
+def expected_turns_overage(result: EvaluationResult) -> tuple[int, int] | None:
+    """Return ``(visible_turns, expected)`` when the visible-events turn
+    count strictly exceeds ``run_limits.expected_turns``; else ``None``.
+
+    Safe against missing ``task_config``, missing ``run_limits``, and
+    non-int ``expected_turns`` values.
+    """
+    task_cfg = result.task_config
+    if task_cfg is None:
+        return None
+    run_limits = (task_cfg.resolved or {}).get("run_limits") or {}
+    if not isinstance(run_limits, dict):
+        return None
+    expected = run_limits.get("expected_turns")
+    if not isinstance(expected, int) or expected < 1:
+        return None
+    actual = visible_turn_count(result)
+    if actual > expected:
+        return actual, expected
+    return None
+
+
+def describe_prompt_config(variant: ExperimentVariant) -> str:
+    """Return a short description of the variant's prompt configuration.
+
+    Returns strings like ``"(base prompt)"``, ``"(prompt override)"``, or
+    ``"(2 mutations: prefix, suffix)"``.
+    """
+    if variant.initial_prompt is not None or variant.initial_prompt_file is not None:
+        return "(prompt override)"
+    if variant.prompt_mutations:
+        type_names = [m.type for m in variant.prompt_mutations]
+        return f"({len(type_names)} mutations: {', '.join(type_names)})"
+    return "(base prompt)"
+
+
+def load_variant_eval_results(
+    run_dir: Path, variant_id: str, task_summaries: list[TaskExperimentSummary]
+) -> list[EvaluationResult]:
+    """Load EvaluationResult objects for a variant from disk.
+
+    Walks all ``<run_dir>/<variant_id>/<task_id>/NN/task.json`` replicate
+    subdirs for each task in ``task_summaries`` and returns every result that
+    loads successfully.
+    """
+    variant_dir = run_dir / variant_id
+    results: list[EvaluationResult] = []
+
+    if not variant_dir.is_dir():
+        return results
+
+    for ts in task_summaries:
+        task_dir = variant_dir / ts.task_id
+        if not task_dir.is_dir():
+            continue
+        for rep_subdir in sorted(task_dir.glob("[0-9][0-9]")):
+            task_json = rep_subdir / "task.json"
+            if task_json.exists():
+                try:
+                    results.append(EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8")))
+                except Exception:
+                    logger.warning("Failed to load %s for variant report", task_json, exc_info=True)
+
+    return results

coder_eval/resources/__init__.py

@@ -0,0 +1,137 @@
+"""Resource management for default configurations."""
+
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+
+@lru_cache(maxsize=1)
+def load_default_ignore_patterns() -> set[str]:
+    """Load default ignore patterns from YAML resource file.
+
+    Returns:
+        Set of pattern strings to ignore during file operations.
+
+    Note:
+        Result is cached for performance. Changes to the YAML file
+        require process restart to take effect.
+    """
+    resource_dir = Path(__file__).parent
+    patterns_file = resource_dir / "default_ignore_patterns.yaml"
+
+    if not patterns_file.exists():
+        # Fallback to hardcoded patterns if file missing
+        return {
+            ".venv",
+            "venv",
+            "ENV",
+            "env",
+            ".git",
+            ".svn",
+            "__pycache__",
+            ".pytest_cache",
+            "*.pyc",
+            "*.egg-info",
+            "dist",
+            "build",
+            "node_modules",
+            ".DS_Store",
+            "Thumbs.db",
+        }
+
+    with open(patterns_file, encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+
+    # Flatten all categories into a single set
+    patterns = set()
+    for category in data.values():
+        if isinstance(category, list):
+            patterns.update(category)
+
+    return patterns
+
+
+def normalize_ignore_pattern_entry(raw: str) -> str:
+    """Strip whitespace from an ignore-pattern entry and validate its shape.
+
+    Returns the cleaned entry. Raises ``ValueError`` on empty / whitespace-only
+    entries and on bare ``"!"`` (negation with no target). Shared by the
+    runtime resolver (:func:`get_ignore_patterns`) and Pydantic
+    ``field_validator`` hooks on ``SandboxConfig.ignore_patterns`` and
+    ``AgentConfig.ignore_patterns`` so malformed YAML fails at load time
+    rather than mid-run.
+    """
+    entry = raw.strip()
+    if not entry:
+        raise ValueError(f"ignore pattern entry is empty or whitespace-only: {raw!r}")
+    if entry == "!":
+        raise ValueError("ignore pattern entry '!' is bare negation with no target")
+    return entry
+
+
+def get_ignore_patterns(additional_patterns: list[str] | None = None) -> set[str]:
+    """Get ignore patterns with optional additions and negations.
+
+    Args:
+        additional_patterns: Optional list of pattern overrides. Plain entries
+            are added on top of the defaults; entries prefixed with ``!`` are
+            removed from the defaults (gitignore-style negation). Example:
+            ``["!dist", "!node_modules", "*.bak"]`` un-ignores ``dist`` and
+            ``node_modules`` (so vendored JS build outputs survive sandbox
+            copy) and additionally ignores ``*.bak`` files.
+
+    Returns:
+        Combined set of default and additional patterns, with negations
+        applied to the defaults.
+
+    Raises:
+        ValueError: If any entry fails :func:`normalize_ignore_pattern_entry`.
+    """
+    patterns = load_default_ignore_patterns().copy()
+
+    if additional_patterns:
+        for raw in additional_patterns:
+            entry = normalize_ignore_pattern_entry(raw)
+            if entry.startswith("!"):
+                patterns.discard(entry[1:])
+            else:
+                patterns.add(entry)
+
+    return patterns
+
+
+def matches_pattern(path_part: str, pattern: str) -> bool:
+    """Check if a path part matches an ignore pattern.
+
+    Supports:
+    - Exact match: "node_modules"
+    - Prefix wildcard: "*.pyc" (matches "file.pyc")
+    - Suffix wildcard: "test_*" (matches "test_utils.py")
+
+    Args:
+        path_part: Single path component (e.g., "file.pyc")
+        pattern: Pattern to match against
+
+    Returns:
+        True if path_part matches pattern
+    """
+    if pattern.startswith("*"):
+        return path_part.endswith(pattern[1:])
+    elif pattern.endswith("*"):
+        return path_part.startswith(pattern[:-1])
+    else:
+        return path_part == pattern
+
+
+def should_ignore_path(path: Path, patterns: set[str]) -> bool:
+    """Check if any part of a path matches ignore patterns.
+
+    Args:
+        path: Path to check
+        patterns: Set of patterns to match against
+
+    Returns:
+        True if any part of the path matches any pattern
+    """
+    return any(any(matches_pattern(str(part), pattern) for pattern in patterns) for part in path.parts)

coder_eval/resources/default_experiment.yaml

@@ -0,0 +1,85 @@
+# Default experiment — provides baseline agent configuration.
+#
+# Tasks that omit the `agent` field inherit these defaults via the
+# 4-layer merge chain: default.yaml → experiment defaults → task YAML → variant.
+# All agent properties are listed explicitly so users know the baseline.
+
+experiment_id: default
+description: "Default experiment - provides baseline agent configuration"
+
+defaults:
+  # Number of replicates for each task (repeats the same task N times per variant).
+  # Useful for quantifying variance in non-deterministic agent behavior.
+  # Overridable per-variant or via --repeats on the CLI.
+  # repeats: 1
+
+  # Run-time caps (turns, wall-clock, tokens, USD). Override per-variant or per-task via
+  # field-merge — a task or variant can set a single key without replacing the whole block.
+  # Per-row dataset multiplication: a 100-row task with max_usd: 0.10 permits up to
+  # $10 cumulative spend.
+  run_limits:
+    # Maximum agent inner-loop turns per iteration (null = SDK default).
+    # Typical range in tasks: 3-18 depending on complexity.
+    max_turns: 20
+    # Soft target: when cumulative SDK turns across all iterations of a task
+    # exceed this, the orchestrator logs a one-shot warning and the report
+    # surfaces a badge. Does NOT abort the run (max_turns is the hard cap).
+    # expected_turns: 15
+    # Maximum total seconds for the entire task evaluation (all iterations combined).
+    task_timeout: 600
+    # Maximum seconds per agent turn / communicate call (null = no limit).
+    turn_timeout: 300
+    # Optional token / cost caps (leave commented to disable).
+    # max_input_tokens: 100000
+    # max_output_tokens: 20000
+    # max_total_tokens: 120000
+    # max_usd: 0.50
+    # count_cached_input: false
+
+  agent:
+    # Agent type (currently only "claude-code" is supported)
+    type: claude-code
+
+    # Permission mode for agent actions:
+    #   default        - ask for confirmation on each action
+    #   acceptEdits    - auto-accept file edits, confirm other actions
+    #   plan           - read-only, no file modifications allowed
+    #   bypassPermissions - auto-accept all actions (use with caution)
+    permission_mode: acceptEdits
+
+    # Specific model to use (null = defer to Claude Code CLI default)
+    # Examples: claude-sonnet-4-6, claude-opus-4-6
+    model: claude-sonnet-4-6
+
+    # Allowed tools — restrict which tools the agent can use (null = all tools)
+    # Most tasks use ["Bash"] or ["Bash", "Read", "Write"]. This default
+    # provides the common set needed for general-purpose coding tasks.
+    allowed_tools: ["Bash", "Read", "Write", "Edit", "Glob", "Grep", "Skill"]
+
+    # Claude Code plugins to load (null = none)
+    # Example: [{"type": "local", "path": "/path/to/skills"}]
+    plugins: null
+
+    # Additional patterns to ignore during file change detection (beyond defaults)
+    # Example: ["*.log", "__pycache__"]
+    ignore_patterns: []
+
+    # Claude Code SDK pass-through options — anything coder_eval doesn't own
+    # directly (e.g. `effort`, `max_thinking_tokens`). Keys must be
+    # ClaudeAgentOptions fields and must not be framework-managed (model,
+    # allowed_tools, permission_mode, hooks, mcp_servers, ...). Deep-merged
+    # across experiment / task / variant / --sdk-option layers, so a higher
+    # layer adding one key doesn't wipe the rest.
+    # Example: sdk_options: {effort: high}
+    # sdk_options: {}
+
+  # Sandbox-internal cleanup appended after each task's own post_run (MST-9674).
+  # PR #250 sandbox-scopes Node/npm installs to <sandbox>/{node_modules,.npm-prefix};
+  # removing them keeps preserved-sandbox artifacts small and is a no-op in normal
+  # (non-preserved) runs where the tempdir is rmtree'd anyway.
+  post_run:
+    - command: "rm -rf node_modules .npm-prefix"
+      timeout: 30
+
+variants:
+  - variant_id: default

coder_eval/resources/default_ignore_patterns.yaml

@@ -0,0 +1,60 @@
+# Default file/directory patterns to ignore during file operations
+# These patterns apply to:
+# - Template directory copying (sandbox setup)
+# - Workspace copying into an agent_judge sub-agent sandbox
+
+# Virtual environments
+virtual_environments:
+  - ".venv"
+  - "venv"
+  - "ENV"
+  - "env"
+  - "virtualenv"
+
+# Version control
+version_control:
+  - ".git"
+  - ".svn"
+  - ".hg"
+
+# Python artifacts
+python_artifacts:
+  - "__pycache__"
+  - "*.pyc"
+  - "*.pyo"
+  - "*.pyd"
+  - ".pytest_cache"
+  - ".ruff_cache"
+  - "*.egg-info"
+  - ".eggs"
+  - "dist"
+  - "build"
+
+# IDE and editor files
+ide_files:
+  - ".vscode"
+  - ".idea"
+  - "*.swp"
+  - "*.swo"
+  - "*~"
+  - ".DS_Store"
+  - "Thumbs.db"
+
+# Package managers
+package_managers:
+  - "node_modules"
+  - ".npm"
+  - "bower_components"
+
+# Test and coverage
+test_coverage:
+  - ".coverage"
+  - "htmlcov"
+  - ".tox"
+  - ".nox"
+
+# Compiled files
+compiled:
+  - "*.so"
+  - "*.dylib"
+  - "*.dll"

coder_eval/resources/tags.yaml

@@ -0,0 +1,55 @@
+# Suggested vocabulary of tags for post-run task reviews.
+#
+# This file is a *suggested* vocabulary, not a strict allowlist. The
+# /coder-eval-review skill prefers names from this list but emits whatever
+# fits. A periodic vocabulary lint reports tags appearing in review.json
+# files that aren't here, so drift is visible
+# without blocking new categories from emerging.
+#
+# Tags can describe failure modes, agent behaviors, or positive traits
+# (e.g. optimal-path, called-tools). To add a vocabulary entry, open a PR.
+#
+# Each entry:
+#   name        - kebab-case canonical slug (required)
+#   definition  - one short sentence describing what the tag means (required)
+#   examples    - optional list of short, illustrative snippets
+tags:
+  - name: validation-error
+    definition: Schema or CLI validation failed for a generated artifact (e.g. .flow, .xaml, JSON).
+    examples:
+      - "Schema validation failed: expected string, received undefined"
+      - "Invalid .flow: missing required field 'startNode'"
+  - name: sandbox-leak
+    definition: Agent read or wrote files outside its assigned task directory.
+    examples:
+      - "Read sibling runs/<other_run>/artifacts/..."
+  - name: agent-loop
+    definition: Agent retried the same failing action repeatedly without adapting.
+    examples:
+      - "Same Bash command failed 5 times in a row with identical error"
+  - name: tool-error
+    definition: Bash/Read/Edit/SDK tool failed for reasons unrelated to task logic.
+    examples:
+      - "Edit failed: file not found"
+      - "Bash: command not found"
+  - name: model-nondeterminism
+    definition: Same task flips pass/fail across replicates with no code change.
+  - name: judge-disagreement
+    definition: LLM-judge verdict inconsistent with artifacts produced.
+  - name: infra
+    definition: Sandbox, network, auth, or external-service failure unrelated to the agent.
+    examples:
+      - "Auth token expired mid-run"
+      - "Network timeout reaching external service"
+  - name: criteria-bug
+    definition: Agent produced correct output but criteria did not match (false negative).
+    examples:
+      - "command_executed expected 'uip run' but agent used 'uipath run'"
+  - name: turn-timeout-too-short
+    definition: Task hit its per-turn timeout; the limit is too tight for the work the task requires.
+    examples:
+      - "300s turn_timeout exceeded at 80 turns; analysis recommends 600s"
+  - name: max-turns-too-low
+    definition: Task ran out of turns before completing; max_turns budget is below what the task realistically needs.
+    examples:
+      - "MAX_TURNS_EXHAUSTED: max_turns=50 but 86 turns used"