agentevolve-optimizer 0.5.0__py3-none-any.whl

agent_evolve/core/problem.py

@@ -0,0 +1,167 @@
+"""Problem protocol, objective specification, and validation outcome."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from dataclasses import dataclass
+from numbers import Real
+from typing import Any, Dict, Literal, Optional, Protocol, Sequence, TypeVar, runtime_checkable
+
+Goal = Literal["min", "max"]
+ConfigT = TypeVar("ConfigT")
+
+
+@dataclass(frozen=True)
+class ObjectiveSpec:
+    """Specification for a single optimisation objective.
+
+    ``description`` is the objective's MEANING -- what the number measures,
+    its units, what a good value looks like ("spec-attainment reward, sum of
+    nine clipped terms, maximised at 0 = every spec met"). It is rendered
+    into every model-facing prompt: an optimizer asked to trade objectives
+    it cannot interpret is reasoning blindfolded, and the resulting failure
+    is unattributable (channel defect vs capability). Optional so existing
+    problems keep working; a problem that leaves it empty is telling the
+    model "the name is all you get".
+    """
+
+    name: str
+    goal: Goal
+    description: str = ""
+
+
+class ProblemContractError(RuntimeError):
+    """The problem adapter violated its declared objective/evaluation contract."""
+
+
+def validate_objective_specs(objectives: Sequence[ObjectiveSpec]) -> None:
+    """Validate objective declarations before any proposal or evaluation work."""
+    if not objectives:
+        raise ProblemContractError("Problem must define at least one objective")
+    names = [spec.name for spec in objectives]
+    if any(not isinstance(name, str) or not name.strip() for name in names):
+        raise ProblemContractError("Objective names must be non-empty strings")
+    duplicates = sorted({name for name in names if names.count(name) > 1})
+    if duplicates:
+        raise ProblemContractError(f"Duplicate objective name(s): {', '.join(duplicates)}")
+    invalid_goals = [f"{spec.name}={spec.goal!r}" for spec in objectives
+                     if spec.goal not in ("min", "max")]
+    if invalid_goals:
+        raise ProblemContractError(
+            "Objective goals must be 'min' or 'max': " + ", ".join(invalid_goals)
+        )
+
+
+def normalize_objective_values(
+    values: Any,
+    objectives: Sequence[ObjectiveSpec],
+) -> Dict[str, float]:
+    """Return a complete finite objective vector or raise ``ProblemContractError``.
+
+    Evaluators must return exactly the declared objectives. Diagnostics belong in
+    a separate adapter-level artifact/metadata channel; accepting undeclared keys
+    here would make misspelled objective names too easy to overlook.
+    """
+    validate_objective_specs(objectives)
+    if not isinstance(values, Mapping):
+        raise ProblemContractError(
+            f"Problem.evaluate() must return a mapping, got {type(values).__name__}"
+        )
+
+    expected = {spec.name for spec in objectives}
+    actual = set(values.keys())
+    missing = sorted(expected - actual)
+    extra = sorted(str(key) for key in actual - expected)
+    if missing or extra:
+        details = []
+        if missing:
+            details.append("missing: " + ", ".join(missing))
+        if extra:
+            details.append("undeclared: " + ", ".join(extra))
+        raise ProblemContractError("Invalid objective mapping (" + "; ".join(details) + ")")
+
+    normalized: Dict[str, float] = {}
+    for spec in objectives:
+        value = values[spec.name]
+        if isinstance(value, bool) or not isinstance(value, Real):
+            raise ProblemContractError(
+                f"Objective {spec.name!r} must be a real number, got {type(value).__name__}"
+            )
+        number = float(value)
+        if not math.isfinite(number):
+            raise ProblemContractError(
+                f"Objective {spec.name!r} must be finite, got {number!r}"
+            )
+        normalized[spec.name] = number
+    return normalized
+
+
+@dataclass(frozen=True)
+class ValidationOutcome:
+    """Structured result of a feasibility pre-check.
+
+    ``failure_phase`` lets a problem label *where* a candidate broke (e.g.
+    ``"structural" | "constraint" | "simulation"``) so the loop can feed richer
+    failure context to the LLM. ``message`` is forwarded verbatim to the model.
+    """
+
+    ok: bool
+    failure_phase: Optional[str] = None
+    message: Optional[str] = None
+
+
+@runtime_checkable
+class Problem(Protocol[ConfigT]):
+    """Minimal interface that every optimisation problem must satisfy.
+
+    Required
+    --------
+    objectives : Sequence[ObjectiveSpec]
+        The objectives to optimise (at least one).
+    evaluate(config) -> Dict[str, float]
+        Return exactly one finite numeric value for every declared objective.
+        Raise ``ValueError`` with a descriptive message only for invalid /
+        infeasible configurations -- the message is forwarded to the LLM as
+        feedback. Infrastructure and programming errors must use other exception
+        types and abort the run rather than becoming candidate feedback.
+
+    Optional (detected via ``hasattr`` at runtime)
+    ----------------------------------------------
+    validate_detailed(config) -> ValidationOutcome
+        Structured feasibility pre-check carrying a ``failure_phase`` label.
+    validate(config) -> bool
+        Legacy boolean pre-check. **Raise** ``ValueError("...")`` when invalid
+        (never return ``False`` silently). Wrapped into a ``ValidationOutcome``.
+    search_space_description() -> str
+        Human-readable description of the configuration format, valid ranges,
+        and constraints. Included verbatim in LLM prompts.
+    render_candidate(config) -> str
+        Compact one-line summary of a configuration, used in failure / Pareto
+        lists shown to the LLM. Defaults to pretty JSON when absent.
+    candidate_key(config) -> str
+        Canonical key used to de-duplicate proposed candidates across the run
+        (so identical configs are not re-evaluated). Defaults to sorted JSON.
+
+    Optional attribute:
+
+    directives
+        A ``Directives`` provider supplying prompt wording for this problem.
+        When absent, the backbone's generic ``DefaultDirectives`` is used.
+
+    Optional attributes (not part of the protocol check):
+
+    candidate_model : type[pydantic.BaseModel]
+        Schema for one candidate; its JSON schema is shown to the LLM.
+    constraints_description : str
+        Extra free-text constraints injected into prompts.
+    example_config : dict
+        A reference configuration the model can imitate.
+    config_schema : dict
+        A pseudo-JSON schema for the configuration.
+    """
+
+    @property
+    def objectives(self) -> Sequence[ObjectiveSpec]: ...
+
+    def evaluate(self, config: ConfigT) -> Dict[str, float]: ...

agent_evolve/core/results.py

@@ -0,0 +1,323 @@
+"""Result containers and Pareto-front utilities."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from numbers import Real
+from typing import Any, Dict, Generic, List, Optional, Sequence, Tuple, TypeVar
+
+from agent_evolve.core.problem import ObjectiveSpec, ProblemContractError
+from agent_evolve.core.telemetry import RunTelemetry
+
+ConfigT = TypeVar("ConfigT")
+
+
+
+@dataclass(frozen=True)
+class ProviderUsageSummary:
+    """What the run spent, so a caller can answer "what did this cost me".
+
+    Counted from the calls actually made, never declared. ``calls`` is zero for
+    an uninformed proposer, and that zero is measured: a run that made no model
+    call still reports the block rather than omitting it, because an absent
+    field cannot be told apart from an unrecorded one.
+    """
+
+    calls: int = 0
+    input_tokens: Optional[int] = None
+    output_tokens: Optional[int] = None
+    cost_usd: Optional[str] = None
+    model: Optional[str] = None
+    reported_by: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        if type(self.calls) is not int or self.calls < 0:
+            raise ValueError("calls must be a non-negative integer")
+        figures = {
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "cost_usd": self.cost_usd,
+        }
+        present = {name for name, value in figures.items() if value is not None}
+        if present and not self.reported_by:
+            # A figure with no reporter is a number nobody measured. Zero is the
+            # dangerous case: it reads as "nothing was spent" when it means "no
+            # one looked". Naming the reporter is what makes the difference
+            # unrepresentable rather than merely documented.
+            raise ValueError(
+                f"usage figures {sorted(present)} require reported_by naming "
+                "what measured them"
+            )
+        if self.reported_by is not None and not present:
+            raise ValueError(
+                "reported_by names a reporter that supplied no figure"
+            )
+
+    @property
+    def provider_free(self) -> bool:
+        """True only when calls were counted and none occurred."""
+
+        return self.calls == 0
+
+    @property
+    def cost_is_known(self) -> bool:
+        return self.cost_usd is not None
+
+
+@dataclass(frozen=True)
+class Candidate(Generic[ConfigT]):
+    """A single evaluated configuration."""
+
+    configuration: ConfigT
+    objectives: Dict[str, float]
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class SearchResult(Generic[ConfigT]):
+    """Aggregated output of an optimisation run.
+
+    Attributes
+    ----------
+    objectives : objectives used during the run.
+    best : the single recommended candidate (minimax rank over the Pareto front;
+        see :func:`select_minimax_rank`).
+    pareto_front : non-dominated set.
+    all_candidates : every unique candidate processed across all generations,
+        including deterministic validation failures.
+    history : per-generation summary dicts.
+    best_per_generation : minimax-best candidate on the cumulative Pareto front
+        after each generation (same rule as ``best``); useful for progress.
+    evaluations : exact number of ``Problem.evaluate`` invocations, including
+        calls that raise candidate-level ``ValueError``. Deterministic pre-check
+        failures do not increment this evaluator-call budget.
+    telemetry : what each guidance mechanism did, plus the real/virtual
+        evaluation ledger. Populated by the genetic loop; ``None`` on paths
+        that have not adopted it yet.
+    """
+
+    objectives: Sequence[ObjectiveSpec]
+    best: Candidate[ConfigT]
+    pareto_front: List[Candidate[ConfigT]] = field(default_factory=list)
+    all_candidates: List[Candidate[ConfigT]] = field(default_factory=list)
+    history: List[Dict[str, Any]] = field(default_factory=list)
+    best_per_generation: List[Candidate[ConfigT]] = field(default_factory=list)
+    evaluations: int = 0
+    provider_usage: "ProviderUsageSummary | None" = None
+    telemetry: Optional[RunTelemetry] = None
+
+    def candidates_by_author(self) -> Dict[str, int]:
+        """How many candidates each authoring call produced.
+
+        Publish this beside any comparison that treats "the model proposed it"
+        as an arm. A count is checkable; a label is only asserted, and an arm
+        named for the component that produced it rather than the authority that
+        decided it is a mistake that survives review because the numbers still
+        look plausible.
+        """
+        counts: Dict[str, int] = {}
+        for candidate in self.all_candidates:
+            author = candidate.metadata.get("authored_by", "unrecorded")
+            counts[author] = counts.get(author, 0) + 1
+        return counts
+
+    def proposed_candidates(self) -> List[Candidate[ConfigT]]:
+        """Only what the proposer authored: the caller's own seeds excluded."""
+
+        return [
+            candidate
+            for candidate in self.all_candidates
+            if candidate.metadata.get("authored_by", "").startswith("proposer_")
+        ]
+
+
+# ------------------------------------------------------------------
+# Pareto dominance
+# ------------------------------------------------------------------
+
+def objective_value(values: Dict[str, float], name: str) -> float:
+    """Read one required finite objective without fabricating a default value."""
+    if name not in values:
+        raise ProblemContractError(f"Missing declared objective {name!r}")
+    value = values[name]
+    if isinstance(value, bool) or not isinstance(value, Real):
+        raise ProblemContractError(
+            f"Objective {name!r} must be a real number, got {type(value).__name__}"
+        )
+    number = float(value)
+    if not math.isfinite(number):
+        raise ProblemContractError(f"Objective {name!r} must be finite, got {number!r}")
+    return number
+
+def dominates(
+    a: Dict[str, float],
+    b: Dict[str, float],
+    objectives: Sequence[ObjectiveSpec],
+) -> bool:
+    """Return *True* if objective vector *a* Pareto-dominates *b*."""
+    all_geq = True
+    any_better = False
+    for spec in objectives:
+        va = objective_value(a, spec.name)
+        vb = objective_value(b, spec.name)
+        if spec.goal == "max":
+            if va < vb:
+                all_geq = False
+            elif va > vb:
+                any_better = True
+        else:
+            if va > vb:
+                all_geq = False
+            elif va < vb:
+                any_better = True
+    return all_geq and any_better
+
+
+def compute_pareto_front(
+    candidates: Sequence[Candidate[ConfigT]],
+    objectives: Sequence[ObjectiveSpec],
+) -> List[Candidate[ConfigT]]:
+    """Return the non-dominated subset of *candidates*.
+
+    Exact duplicates — same configuration identity and same measured
+    objectives — collapse to their first occurrence, so a configuration a
+    population re-visits across generations appears once on the front rather
+    than once per visit. A configuration re-evaluated to *different*
+    objectives is a genuinely different measurement and both rows remain;
+    dropping one silently would be the library's judgement, not the caller's.
+    """
+    if not candidates:
+        return []
+    from agent_evolve.contract import artifact_key
+
+    unique: List[Candidate[ConfigT]] = []
+    seen: set = set()
+    for c in candidates:
+        key = (artifact_key(c.configuration), tuple(sorted(c.objectives.items())))
+        if key in seen:
+            continue
+        seen.add(key)
+        unique.append(c)
+    front: List[Candidate[ConfigT]] = []
+    for i, c in enumerate(unique):
+        if not any(
+            dominates(other.objectives, c.objectives, objectives)
+            for j, other in enumerate(unique)
+            if j != i
+        ):
+            front.append(c)
+    return front
+
+
+# ------------------------------------------------------------------
+# Best-candidate selection
+# ------------------------------------------------------------------
+
+def select_best_candidate(
+    pareto: Sequence[Candidate[ConfigT]],
+    objectives: Sequence[ObjectiveSpec],
+    priority_order: Optional[List[str]] = None,
+) -> Optional[Candidate[ConfigT]]:
+    """Lexicographic selection from the Pareto front.
+
+    Default priority: maximise objectives first, then minimise objectives.
+    """
+    if not pareto:
+        return None
+    if priority_order is None:
+        max_objs = [s for s in objectives if s.goal == "max"]
+        min_objs = [s for s in objectives if s.goal == "min"]
+        priority_order = [s.name for s in max_objs] + [s.name for s in min_objs]
+    obj_map = {s.name: s for s in objectives}
+    unknown = [name for name in priority_order if name not in obj_map]
+    if unknown:
+        raise ProblemContractError(
+            "Unknown priority objective name(s): " + ", ".join(unknown)
+        )
+
+    def _key(c: Candidate[ConfigT]) -> Tuple[float, ...]:
+        parts: List[float] = []
+        for name in priority_order:
+            spec = obj_map[name]
+            val = objective_value(c.objectives, name)
+            parts.append(-val if spec.goal == "max" else val)
+        return tuple(parts)
+
+    return min(pareto, key=_key)
+
+
+# ------------------------------------------------------------------
+# Minimax-rank selection
+# ------------------------------------------------------------------
+
+def _rank_candidates(
+    candidates: Sequence[Candidate[ConfigT]],
+    objectives: Sequence[ObjectiveSpec],
+) -> List[List[int]]:
+    """``ranks[i][j]`` = 1-based **dense** rank of candidate *i* on objective *j* (1 = best).
+
+    Ties share the same rank; the next distinct value gets the next integer (no gaps
+    from skipped positions).
+    """
+    n = len(candidates)
+    ranks: List[List[int]] = [[0] * len(objectives) for _ in range(n)]
+    for j, spec in enumerate(objectives):
+        values = [objective_value(c.objectives, spec.name) for c in candidates]
+        reverse = spec.goal == "max"
+        order = sorted(range(n), key=lambda i: values[i], reverse=reverse)
+        dense = 1
+        for pos, idx in enumerate(order):
+            if pos > 0 and values[order[pos]] != values[order[pos - 1]]:
+                dense += 1
+            ranks[idx][j] = dense
+    return ranks
+
+
+def select_minimax_rank(
+    candidates: Sequence[Candidate[ConfigT]],
+    objectives: Sequence[ObjectiveSpec],
+) -> Optional[Candidate[ConfigT]]:
+    """Best pick for multi-objective summaries: minimax over per-objective ranks.
+
+    For each candidate, compute **dense** rank on each objective (1 = best among
+    *candidates*). Take the **maximum** rank across objectives (bottleneck / worst
+    placement). Prefer the candidate(s) with the **smallest** bottleneck (minimax).
+
+    If several tie, pick the one with the **smallest sum of ranks** (more uniform
+    strength, not a spike on one metric).
+    """
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        return candidates[0]
+    ranks = _rank_candidates(candidates, objectives)
+    worst = [max(r) for r in ranks]
+    min_worst = min(worst)
+    tied = [i for i in range(len(candidates)) if worst[i] == min_worst]
+    if len(tied) == 1:
+        return candidates[tied[0]]
+    best_idx = min(tied, key=lambda i: sum(ranks[i]))
+    return candidates[best_idx]
+
+
+def sort_by_minimax_rank(
+    candidates: Sequence[Candidate[ConfigT]],
+    objectives: Sequence[ObjectiveSpec],
+) -> List[Candidate[ConfigT]]:
+    """Order *candidates* by the same rule as :func:`select_minimax_rank`.
+
+    Primary key: smallest worst per-objective (dense) rank. Secondary: smallest
+    sum of per-objective ranks. The first element matches what
+    ``select_minimax_rank(candidates, objectives)`` returns when not ``None``.
+    """
+    if not candidates:
+        return []
+    if len(candidates) == 1:
+        return [candidates[0]]
+    ranks = _rank_candidates(candidates, objectives)
+    order = sorted(
+        range(len(candidates)),
+        key=lambda i: (max(ranks[i]), sum(ranks[i])),
+    )
+    return [candidates[i] for i in order]

agent_evolve/core/stats.py

@@ -0,0 +1,70 @@
+"""Performance statistics and failure sampling for the insight prompts."""
+
+from __future__ import annotations
+
+import random
+from typing import Any, Dict, List, Optional, Sequence
+
+from agent_evolve.core.formatting import (
+    CandidateResult,
+    candidate_to_result,
+    result_to_candidate,
+)
+from agent_evolve.core.problem import ObjectiveSpec
+from agent_evolve.core.results import compute_pareto_front, objective_value, sort_by_minimax_rank
+
+
+def compute_performance_stats(
+    valid_results: Sequence[CandidateResult],
+    objectives: Sequence[ObjectiveSpec],
+) -> Optional[Dict[str, Any]]:
+    """Compute best/worst per objective and the top Pareto candidates."""
+    if not valid_results:
+        return None
+
+    stats: Dict[str, Any] = {}
+
+    for spec in objectives:
+        key = spec.name
+        if spec.goal == "max":
+            best = max(valid_results, key=lambda r: objective_value(r.objectives, key))
+            worst = min(valid_results, key=lambda r: objective_value(r.objectives, key))
+        else:
+            best = min(valid_results, key=lambda r: objective_value(r.objectives, key))
+            worst = max(valid_results, key=lambda r: objective_value(r.objectives, key))
+        stats[f"best_{key}"] = best
+        stats[f"worst_{key}"] = worst
+
+    candidates = [result_to_candidate(r) for r in valid_results]
+    pareto_candidates = compute_pareto_front(candidates, objectives)
+    sorted_pareto = sort_by_minimax_rank(pareto_candidates, objectives)
+    pareto_results = [candidate_to_result(c) for c in sorted_pareto]
+    stats["top_3_pareto"] = pareto_results[:3]
+    stats["pareto_front"] = pareto_results
+    stats["pareto_size"] = len(pareto_results)
+
+    return stats
+
+
+def sample_failed_for_constraint(
+    latest_failed: Sequence[CandidateResult],
+    all_previous_failed: Sequence[CandidateResult],
+    max_examples: int,
+    rng: Optional[random.Random] = None,
+) -> List[CandidateResult]:
+    """Sample failures for constraint-instruction generation.
+
+    Always includes the latest failures; fills remaining slots with random
+    previous failures using the injected ``rng`` for reproducibility.
+    """
+    sampled = list(latest_failed)
+    if len(sampled) >= max_examples:
+        return sampled[:max_examples]
+
+    remaining = max_examples - len(sampled)
+    latest_ids = {id(r) for r in latest_failed}
+    previous = [r for r in all_previous_failed if id(r) not in latest_ids]
+    if previous and remaining > 0:
+        picker = rng or random
+        sampled.extend(picker.sample(previous, min(remaining, len(previous))))
+    return sampled

agent_evolve/core/telemetry.py

@@ -0,0 +1,100 @@
+"""What each guidance mechanism actually did, surfaced with the result.
+
+The seam objects count their own behaviour (calls, rejections, authoring
+attempts) and always have — but the counters lived as attributes on closures
+the caller discarded, so no run could report what its guidance did. A number
+that is counted but unreachable might as well not exist: the difference
+between "guidance did not help" and "guidance never arrived" is exactly these
+counters, and it must be readable off the ``SearchResult``.
+
+The contract is deliberately small. Any seam object may carry:
+
+``telemetry``    an object whose ``as_dict()`` returns integer counters;
+``mechanism``    a short name for what the seam decides (``"chooser"``);
+``authored_by``  who authored the decisions — ``"llm"``, ``"rule"``, or
+                 ``"none"``.
+
+:func:`harvest_telemetry` gathers whatever is present and skips whatever is
+not, so the unguided path reports an empty mechanism list rather than nothing
+at all — a measured zero, distinguishable from "nobody looked".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Iterable, Mapping, Optional, Tuple
+
+__all__ = ["MechanismTelemetry", "RunTelemetry", "harvest_telemetry"]
+
+
+@dataclass(frozen=True)
+class MechanismTelemetry:
+    """One mechanism's counters, labelled with who authored its decisions."""
+
+    mechanism: str
+    authored_by: str
+    counters: Mapping[str, int]
+
+
+@dataclass(frozen=True)
+class RunTelemetry:
+    """A run's mechanism counters plus its evaluation ledger.
+
+    ``real_evaluations`` counts evaluator invocations that were charged
+    against the budget. ``virtual_evaluations`` counts surrogate predictions —
+    free by construction, and reported separately precisely so the two can
+    never be conflated in a budget claim.
+
+    ``proxy_evaluations`` counts calls to a problem's CHEAPER evaluation
+    fidelity (``Problem.evaluate_proxy``). They are not free — they burn real
+    evaluator seconds — and they are not charged: the budget a claim is
+    denominated in counts full-fidelity evaluations. A campaign that spends
+    them must therefore report them, which is why they have a field of their
+    own here rather than a line in someone's log.
+    """
+
+    mechanisms: Tuple[MechanismTelemetry, ...] = ()
+    real_evaluations: int = 0
+    virtual_evaluations: int = 0
+    proxy_evaluations: int = 0
+
+
+def harvest_telemetry(
+    sources: Iterable[Optional[Any]],
+    *,
+    real_evaluations: int = 0,
+    virtual_evaluations: int = 0,
+    proxy_evaluations: int = 0,
+) -> RunTelemetry:
+    """Collect telemetry from whichever *sources* carry it.
+
+    ``None`` entries and objects without a usable ``telemetry.as_dict()`` are
+    skipped silently: absence of counters is a legitimate state (the random
+    chooser, the statistical prior), not an error.
+    """
+
+    mechanisms = []
+    for source in sources:
+        if source is None:
+            continue
+        counter = getattr(source, "telemetry", None)
+        as_dict = getattr(counter, "as_dict", None)
+        if not callable(as_dict):
+            continue
+        counters = {str(k): int(v) for k, v in dict(as_dict()).items()}
+        mechanisms.append(
+            MechanismTelemetry(
+                mechanism=str(
+                    getattr(source, "mechanism", None)
+                    or getattr(source, "__name__", type(source).__name__)
+                ),
+                authored_by=str(getattr(source, "authored_by", "unrecorded")),
+                counters=counters,
+            )
+        )
+    return RunTelemetry(
+        mechanisms=tuple(mechanisms),
+        real_evaluations=int(real_evaluations),
+        virtual_evaluations=int(virtual_evaluations),
+        proxy_evaluations=int(proxy_evaluations),
+    )