agentevolve-optimizer 0.5.0__py3-none-any.whl

agent_evolve/session/evaluate.py

@@ -0,0 +1,236 @@
+"""Black-box candidate evaluation against a Problem.
+
+Runs the three per-candidate obligations in order: ``validate`` rejects cheaply
+and explains itself, ``materialize`` builds the artifact, ``evaluate`` measures
+it. Expected infeasibility surfaces as a failed :class:`CandidateResult`
+carrying a ``failure_phase`` and message; the message is forwarded to the LLM
+as feedback.
+
+Evaluation is cached on the identity of the *materialized artifact*, not the
+configuration. Distinct configurations that build the same artifact are
+therefore measured once. Keying on the configuration -- which is what this did
+before -- paid twice for the redundancy the union-over-sum analysis measured as
+the dominant loss.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, MutableMapping, Optional, Sequence, Tuple
+
+from agent_evolve.contract import artifact_key, as_evaluator
+from agent_evolve.core.formatting import CandidateResult
+from agent_evolve.core.problem import (
+    ObjectiveSpec,
+    ValidationOutcome,
+    normalize_objective_values,
+)
+
+INVALID_PENALTY: float = 1e18
+
+_VALIDATION_FALSE_HINT = (
+    "validate() returned False without a reason. Raise ValueError('...') from "
+    "validate() describing what is wrong and how to fix it (unknown keys, wrong "
+    "types, out-of-range values, invalid combinations)."
+)
+
+
+def format_optimizer_error(exc: BaseException) -> str:
+    """Format an exception for regeneration prompts with a concrete fix hint."""
+    name = type(exc).__name__
+    msg = str(exc).strip()
+    if msg:
+        return f"{name}: {msg}"
+    return (
+        f"{name} was raised with no message. Raise ValueError('clear explanation of "
+        "what failed and valid alternatives') instead."
+    )
+
+
+def _penalty_objectives(objectives: Sequence[ObjectiveSpec]) -> Dict[str, float]:
+    return {s.name: (0.0 if s.goal == "max" else INVALID_PENALTY) for s in objectives}
+
+
+def _failure(
+    config: Dict[str, Any],
+    objectives: Sequence[ObjectiveSpec],
+    message: str,
+    phase: str,
+    *,
+    evaluation_attempted: bool = False,
+) -> CandidateResult:
+    return CandidateResult(
+        configuration=config,
+        objectives=_penalty_objectives(objectives),
+        is_valid=False,
+        error_message=message,
+        failure_phase=phase,
+        evaluation_attempted=evaluation_attempted,
+    )
+
+
+class EvaluationCache(dict):
+    """Artifact identity -> objective vector, plus what it saved.
+
+    ``hits`` is the number of evaluations not paid for. It is reported rather
+    than inferred, so a problem whose ``materialize`` collapses nothing can see
+    that immediately instead of assuming a saving it never got.
+
+    ``misses`` is what the budget actually counts: distinct artifacts measured.
+
+    A REFUSAL is a measurement too. An artifact whose ``evaluate`` raised was
+    charged -- it ran the evaluator, it took the wall-clock -- so its outcome
+    is recorded exactly like a successful one, and a re-proposed refusal is
+    served from memory instead of being charged again. On a venue that
+    refuses over half of what is proposed (the EDA case: 53%), forgetting
+    refusals meant a config could be billed every time the sampler re-drew
+    it. ``refusal_hits`` counts those replays, apart from ``hits``, because
+    "we did not pay for a success again" and "we did not pay for a failure
+    again" are different savings and a campaign should see both.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.hits = 0
+        self.misses = 0
+        self.refusal_hits = 0
+        #: artifact key -> (message, failure phase) of the charged refusal.
+        self._refusals: Dict[str, Tuple[str, str]] = {}
+
+    def record_refusal(self, key: str, message: str, phase: str) -> None:
+        self._refusals[key] = (str(message), str(phase))
+
+    def refusal(self, key: str) -> Optional[Tuple[str, str]]:
+        return self._refusals.get(key)
+
+    #: Hard ceiling on ``misses``. Once reached, no further artifact is
+    #: measured; a budget that is only checked between generations is not a
+    #: budget, and a caller who said 40 must not be billed for 48.
+    budget: Optional[int] = None
+
+    def exhausted(self) -> bool:
+        return self.budget is not None and self.misses >= self.budget
+
+
+def evaluate_batch(
+    problem: Any,
+    candidates: List[Dict[str, Any]],
+    objectives: Sequence[ObjectiveSpec],
+    cache: Optional[MutableMapping[str, Dict[str, float]]] = None,
+) -> Tuple[List[CandidateResult], List[CandidateResult], List[CandidateResult]]:
+    """Evaluate *candidates* against *problem*.
+
+    Returns ``(valid, failed, ordered)`` where *ordered* has one
+    :class:`CandidateResult` per input candidate in input order. When *cache*
+    is supplied, artifacts already measured are not measured again.
+    """
+    bound = as_evaluator(problem)
+    valid: List[CandidateResult] = []
+    failed: List[CandidateResult] = []
+    ordered: List[CandidateResult] = []
+
+    def _record(cr: CandidateResult, bucket: List[CandidateResult]) -> None:
+        bucket.append(cr)
+        ordered.append(cr)
+
+    for config in candidates:
+        try:
+            outcome = bound.validate(config)
+        except ValueError as exc:
+            outcome = ValidationOutcome(False, "validation", format_optimizer_error(exc))
+        if not outcome.ok:
+            _record(
+                _failure(
+                    config,
+                    objectives,
+                    outcome.message or "invalid configuration",
+                    outcome.failure_phase or "validation",
+                ),
+                failed,
+            )
+            continue
+
+        try:
+            artifact = bound.materialize(config)
+        except ValueError as exc:
+            _record(
+                _failure(config, objectives, format_optimizer_error(exc), "materialization"),
+                failed,
+            )
+            continue
+
+        key = artifact_key(artifact) if cache is not None else None
+        if key is not None and isinstance(cache, EvaluationCache):
+            remembered = cache.refusal(key)
+            if remembered is not None:
+                # This artifact was already measured and REFUSED, and that
+                # refusal was charged once. Replay the recorded outcome --
+                # same message, same phase, so the feedback upstream is
+                # identical -- without running the evaluator or charging the
+                # budget again (evaluation_attempted stays False: no
+                # evaluation happened HERE).
+                message, phase = remembered
+                cache.refusal_hits += 1
+                _record(_failure(config, objectives, message, phase), failed)
+                continue
+        if key is not None and key in cache:
+            normalized = dict(cache[key])
+            if isinstance(cache, EvaluationCache):
+                cache.hits += 1
+        elif isinstance(cache, EvaluationCache) and cache.exhausted():
+            # The budget is spent. Report it as a refusal rather than quietly
+            # measuring one more, so the number the caller gave is the number
+            # they are billed for.
+            _record(
+                _failure(
+                    config,
+                    objectives,
+                    f"evaluation budget of {cache.budget} exhausted",
+                    "budget",
+                ),
+                failed,
+            )
+            continue
+        else:
+            try:
+                obj = bound.evaluate(artifact)
+            except ValueError as exc:
+                # A measurement that failed was still a measurement. It ran the
+                # simulator, it took the wall-clock, and on a timeout it took
+                # the *most* wall-clock of anything in the batch. Not charging
+                # it let a run keep going until it had 40 successes, however
+                # many artifacts that took, and report the budget as honoured.
+                if isinstance(cache, EvaluationCache):
+                    cache.misses += 1
+                    if key is not None:
+                        # Remember the refusal under the artifact's identity,
+                        # so re-proposing it is a replay, never a second bill.
+                        cache.record_refusal(
+                            key, format_optimizer_error(exc), "evaluation")
+                _record(
+                    _failure(
+                        config,
+                        objectives,
+                        format_optimizer_error(exc),
+                        "evaluation",
+                        evaluation_attempted=True,
+                    ),
+                    failed,
+                )
+                continue
+            normalized = normalize_objective_values(obj, objectives)
+            if key is not None:
+                cache[key] = dict(normalized)
+                if isinstance(cache, EvaluationCache):
+                    cache.misses += 1
+
+        _record(
+            CandidateResult(
+                configuration=config,
+                objectives=normalized,
+                is_valid=True,
+                evaluation_attempted=True,
+            ),
+            valid,
+        )
+
+    return valid, failed, ordered

agent_evolve/session/fidelity.py

@@ -0,0 +1,237 @@
+"""A problem's optional CHEAPER evaluation fidelity, and its separate ledger.
+
+Some evaluators expose a knob that buys a correlated answer for a fraction of
+the cost -- fewer mapper iterations, a coarser mesh, a shorter simulation, a
+subsample of the workload. A campaign whose budget is denominated in REAL
+evaluations can spend that cheaper mode on evidence the surrogate needs and
+still be honest about what it paid, but only if the two are never added up.
+
+This module is the seam, and its whole design is the separation:
+
+* :class:`ProxySource` holds the problem's ``evaluate_proxy`` bound method and
+  a key function, **and nothing else**. It has no reference to the problem,
+  the evaluation cache, or the budget, so "a proxy evaluation cannot become a
+  charged evaluation" is a property of the object graph rather than a promise
+  in a docstring -- the same construction that makes the screen unable to
+  spend budget (see :mod:`agent_evolve.session.screening`).
+* Every proxy call is counted in :class:`ProxyLedger`, reported beside the
+  charged count and never folded into it.
+* A hard ``ceiling`` bounds what a run may spend at the cheap fidelity, so a
+  consumer that asks for a proxy value in a loop degrades to "no proxy" rather
+  than to an unbounded bill.
+
+A problem opts in by defining::
+
+    def evaluate_proxy(self, config) -> Mapping[str, float]: ...
+
+returning EXACTLY the declared objectives, at a cheaper and correlated
+fidelity. Optionally it may describe that fidelity for the record::
+
+    proxy_fidelity_name: str            # e.g. "timeloop@50-mappings"
+    proxy_cost_ratio: float             # measured seconds(proxy)/seconds(real)
+
+Nothing here decides whether a proxy is GOOD. That is the gate's job: a proxy
+enters the screen as evidence or as a candidate ordering and is
+cross-validated against the run's own REAL measurements like anything else.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence
+
+from agent_evolve.core.problem import ObjectiveSpec, normalize_objective_values
+
+__all__ = ["ProxyLedger", "ProxySource", "proxy_fidelity_builder"]
+
+Config = Dict[str, Any]
+
+
+def _canonical_key(config: Mapping[str, Any]) -> str:
+    return json.dumps(config, sort_keys=True, default=str)
+
+
+@dataclass
+class ProxyLedger:
+    """What the cheap fidelity was asked for, and what it cost. Never charged.
+
+    ``evaluations`` counts calls that actually ran the cheap evaluator;
+    ``cache_hits`` counts repeats served from memory, because a consumer that
+    re-asks for the same candidate must not be reported as having spent
+    anything twice. ``refused_ceiling`` is the count of requests declined
+    because the run had spent its proxy allowance.
+    """
+
+    evaluations: int = 0
+    cache_hits: int = 0
+    failures: int = 0
+    refused_ceiling: int = 0
+    rows_offered: int = 0
+    rows_used: int = 0
+
+    def as_dict(self) -> Dict[str, int]:
+        return {
+            "proxy_evaluations": self.evaluations,
+            "proxy_cache_hits": self.cache_hits,
+            "proxy_failures": self.failures,
+            "proxy_refused_ceiling": self.refused_ceiling,
+            "proxy_rows_offered": self.rows_offered,
+            "proxy_rows_used": self.rows_used,
+        }
+
+
+class ProxySource:
+    """The cheap fidelity, memoized, counted, and bounded.
+
+    Construct with :meth:`for_problem`, which returns ``None`` when the
+    problem exposes no cheaper fidelity -- so every consumer's "off" path is
+    the absence of an object rather than a flag it might forget to read.
+    """
+
+    def __init__(
+        self,
+        evaluate: Callable[[Config], Mapping[str, float]],
+        objectives: Sequence[ObjectiveSpec],
+        *,
+        key: Callable[[Config], str] = _canonical_key,
+        name: str = "proxy",
+        cost_ratio: Optional[float] = None,
+        ceiling: Optional[int] = None,
+    ) -> None:
+        if not callable(evaluate):
+            raise TypeError("evaluate must be callable")
+        if ceiling is not None and ceiling < 0:
+            raise ValueError(f"ceiling must not be negative, got {ceiling}")
+        # DELIBERATELY only these: a bound method, a key function, the
+        # objective specs, and counters. No problem, no cache, no budget.
+        self._evaluate = evaluate
+        self._key = key
+        self._specs = list(objectives)
+        self.name = str(name)
+        self.cost_ratio = None if cost_ratio is None else float(cost_ratio)
+        self.ceiling = ceiling
+        self.ledger = ProxyLedger()
+        self._memo: Dict[str, Optional[Dict[str, float]]] = {}
+        #: the harvest contract (`core.telemetry`), so the cheap fidelity's
+        #: ledger reaches the SearchResult like any other mechanism's counters
+        self.telemetry = self.ledger
+        self.mechanism = "proxy_fidelity"
+        self.authored_by = "none"
+
+    # -- construction ------------------------------------------------------
+    @classmethod
+    def for_problem(cls, problem: Any, *, ceiling: Optional[int] = None
+                    ) -> Optional["ProxySource"]:
+        """The problem's cheap fidelity, or ``None`` if it does not have one."""
+
+        evaluate = getattr(problem, "evaluate_proxy", None)
+        if not callable(evaluate):
+            return None
+        key = getattr(problem, "candidate_key", None)
+        return cls(
+            evaluate,
+            list(problem.objectives),
+            key=key if callable(key) else _canonical_key,
+            name=str(getattr(problem, "proxy_fidelity_name", "proxy")),
+            cost_ratio=getattr(problem, "proxy_cost_ratio", None),
+            ceiling=ceiling,
+        )
+
+    # -- use ---------------------------------------------------------------
+    def key(self, config: Config) -> str:
+        return self._key(config)
+
+    def evaluate(self, config: Config) -> Optional[Dict[str, float]]:
+        """The cheap objectives for *config*, or ``None``.
+
+        ``None`` means "this run has no cheap answer for this candidate" --
+        the evaluator refused, returned something the objective contract does
+        not accept, or the proxy allowance is spent. Every consumer treats
+        that as "no proxy", never as a value.
+        """
+
+        token = self._key(config)
+        if token in self._memo:
+            self.ledger.cache_hits += 1
+            return self._memo[token]
+        if self.ceiling is not None and self.ledger.evaluations >= self.ceiling:
+            self.ledger.refused_ceiling += 1
+            return None
+        self.ledger.evaluations += 1
+        try:
+            values = normalize_objective_values(self._evaluate(config), self._specs)
+        except Exception:
+            self.ledger.failures += 1
+            self._memo[token] = None
+            return None
+        self._memo[token] = values
+        return values
+
+    def rows(
+        self, candidates: Sequence[Config], *, exclude: Sequence[str] = ()
+    ) -> list[tuple[Config, Dict[str, float]]]:
+        """``(config, cheap objectives)`` for each candidate that yields one.
+
+        ``exclude`` names candidate keys that already have a REAL measurement.
+        A real row always supersedes a cheap one: the point of the seam is to
+        add evidence where there is none, never to dilute evidence there is.
+        """
+
+        skip = set(exclude)
+        out = []
+        for candidate in candidates:
+            if self._key(candidate) in skip:
+                continue
+            self.ledger.rows_offered += 1
+            values = self.evaluate(candidate)
+            if values is not None:
+                out.append((dict(candidate), values))
+        return out
+
+
+def proxy_fidelity_builder(source: ProxySource) -> Callable[..., Any]:
+    """The cheap fidelity itself, as a :data:`SurrogateBuilder`.
+
+    A surrogate is anything that ORDERS candidates without charging the
+    budget, and a correlated cheap evaluation is exactly that. Presenting it
+    as a builder means it earns its place the same way an authored artifact
+    does: the screen's gate cross-validates it against the run's own REAL
+    measurements, per objective, and admits it only if it ranks them. Nothing
+    about this asserts that a cheap fidelity ranks well -- it makes the claim
+    checkable in the loop that would rely on it.
+
+    The builder ignores the training rows because there is nothing to fit.
+    """
+
+    def build(rows: Any, specs: Any) -> Callable[[Sequence[Config]], Any]:
+        names = [spec.name for spec in specs]
+        goals = {spec.name: spec.goal for spec in specs}
+
+        def predict(configs: Sequence[Config]) -> Any:
+            out: list[Optional[Dict[str, float]]] = []
+            for config in configs:
+                out.append(source.evaluate(config))
+            got = [row for row in out if row is not None]
+            if not got:
+                return None          # no cheap answer at all -> screen nothing
+            if len(got) == len(out):
+                return out
+            # A candidate the cheap fidelity REFUSES is ranked last, not
+            # dropped: refusing one member of a pool must not blind the screen
+            # to the other n-1. This mirrors what the expensive evaluator
+            # already does with an infeasible candidate -- it returns a
+            # penalty vector rather than nothing -- and it keeps the returned
+            # sequence aligned with the pool the caller passed in.
+            worst: Dict[str, float] = {}
+            for name in names:
+                values = [row[name] for row in got]
+                span = max(values) - min(values)
+                pad = abs(span) * 0.1 + (abs(max(values, key=abs)) * 1e-9) + 1e-12
+                worst[name] = (max(values) + pad if goals.get(name) == "min"
+                               else min(values) - pad)
+            return [dict(worst) if row is None else row for row in out]
+
+        return predict
+
+    return build