agentevolve-optimizer 0.5.0__py3-none-any.whl

agent_evolve/application/evaluation_accounting.py

@@ -0,0 +1,325 @@
+"""Workload-neutral accounting for candidate occurrences and physical evaluations.
+
+An optimizer can administer two distinct candidate occurrences that resolve to
+the same benchmark phenotype.  The evaluation cache should coalesce that work,
+but experiment reports must never silently relabel the cache hit as a fresh
+physical evaluation.  This module validates the complete seed/stage partition
+and publishes both quantities explicitly.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class CampaignEvaluationAccounting:
+    """Exact occurrence/physical-call partition for one completed campaign.
+
+    ``planned_candidate_occurrences`` is an exact commitment by default.  A
+    campaign whose evolutionary operators may *typed-abstain* (for example,
+    when no legal crossover pair exists) can additionally publish a
+    ``minimum_candidate_occurrences``.  In that mode the frozen plan is a hard
+    capacity ceiling and the minimum is the mandatory work floor.  This keeps
+    legal operator abstention distinct from both silent budget loss and cache
+    reuse: every realized occurrence is still partitioned exactly, while the
+    unfilled capacity is reported explicitly.
+    """
+
+    planned_candidate_occurrences: int
+    seed_occurrences: int
+    seed_unique_evaluations: int
+    stage_occurrences: tuple[int, ...]
+    stage_unique_evaluations: tuple[int, ...]
+    candidate_occurrences: int
+    unique_evaluations: int
+    minimum_candidate_occurrences: int | None = None
+
+    def __post_init__(self) -> None:
+        for name in (
+            "planned_candidate_occurrences",
+            "seed_occurrences",
+            "seed_unique_evaluations",
+            "candidate_occurrences",
+            "unique_evaluations",
+        ):
+            value = getattr(self, name)
+            if type(value) is not int or value < 0:
+                raise ValueError(f"{name} must be a non-negative exact integer")
+        if self.minimum_candidate_occurrences is not None and (
+            type(self.minimum_candidate_occurrences) is not int
+            or self.minimum_candidate_occurrences < 0
+        ):
+            raise ValueError(
+                "minimum_candidate_occurrences must be a non-negative exact "
+                "integer or None"
+            )
+        for name in ("stage_occurrences", "stage_unique_evaluations"):
+            values = getattr(self, name)
+            if type(values) is not tuple or any(
+                type(value) is not int or value < 0 for value in values
+            ):
+                raise ValueError(f"{name} must be an exact non-negative tuple")
+        if len(self.stage_occurrences) != len(self.stage_unique_evaluations):
+            raise ValueError("stage occurrence and evaluation tuples must align")
+        if self.seed_unique_evaluations > self.seed_occurrences:
+            raise ValueError("seed physical evaluations exceed seed occurrences")
+        if any(
+            unique > occurrences
+            for occurrences, unique in zip(
+                self.stage_occurrences,
+                self.stage_unique_evaluations,
+                strict=True,
+            )
+        ):
+            raise ValueError("stage physical evaluations exceed occurrences")
+        if self.candidate_occurrences != (
+            self.seed_occurrences + sum(self.stage_occurrences)
+        ):
+            raise ValueError("candidate occurrences do not equal the stage partition")
+        if self.unique_evaluations != (
+            self.seed_unique_evaluations + sum(self.stage_unique_evaluations)
+        ):
+            raise ValueError("unique evaluations do not equal the stage partition")
+        if self.minimum_candidate_occurrences is None:
+            if self.candidate_occurrences != self.planned_candidate_occurrences:
+                raise ValueError("completed occurrences differ from the frozen plan")
+        elif not (
+            self.minimum_candidate_occurrences
+            <= self.candidate_occurrences
+            <= self.planned_candidate_occurrences
+        ):
+            raise ValueError(
+                "completed occurrences escape the frozen capacity envelope"
+            )
+
+    @property
+    def planned_underfill_occurrences(self) -> int:
+        self.__post_init__()
+        return self.planned_candidate_occurrences - self.candidate_occurrences
+
+    @property
+    def candidate_capacity_utilization(self) -> float:
+        self.__post_init__()
+        if self.planned_candidate_occurrences == 0:
+            return 1.0
+        return self.candidate_occurrences / self.planned_candidate_occurrences
+
+    @property
+    def cache_reuse_occurrences(self) -> int:
+        self.__post_init__()
+        return self.candidate_occurrences - self.unique_evaluations
+
+    @property
+    def physical_evaluation_utilization(self) -> float:
+        self.__post_init__()
+        if self.candidate_occurrences == 0:
+            return 1.0
+        return self.unique_evaluations / self.candidate_occurrences
+
+    def within_cache_reuse_limit(self, maximum: int) -> bool:
+        self.__post_init__()
+        if type(maximum) is not int or maximum < 0:
+            raise ValueError("maximum must be a non-negative exact integer")
+        return self.cache_reuse_occurrences <= maximum
+
+    def to_record(self) -> dict[str, object]:
+        self.__post_init__()
+        return {
+            "schema_version": 1,
+            "planned_candidate_occurrences": self.planned_candidate_occurrences,
+            "minimum_candidate_occurrences": self.minimum_candidate_occurrences,
+            "candidate_plan_mode": (
+                "exact"
+                if self.minimum_candidate_occurrences is None
+                else "typed_operator_abstention_capacity_envelope"
+            ),
+            "seed_occurrences": self.seed_occurrences,
+            "seed_unique_evaluations": self.seed_unique_evaluations,
+            "stage_occurrences": list(self.stage_occurrences),
+            "stage_unique_evaluations": list(self.stage_unique_evaluations),
+            "candidate_occurrences": self.candidate_occurrences,
+            "unique_evaluations": self.unique_evaluations,
+            "planned_underfill_occurrences": self.planned_underfill_occurrences,
+            "candidate_capacity_utilization_hex": (
+                self.candidate_capacity_utilization.hex()
+            ),
+            "cache_reuse_occurrences": self.cache_reuse_occurrences,
+            "physical_evaluation_utilization_hex": (
+                self.physical_evaluation_utilization.hex()
+            ),
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class CampaignPortfolioEvidenceAccounting:
+    """Exact evidence partition for finite-portfolio campaign outcomes.
+
+    Evaluator-declared candidate infeasibility is a terminal scientific
+    outcome, but it cannot publish objective deltas or forecast-calibration
+    observations.  This accounting keeps those candidates in the proposal and
+    feedback action budget without fabricating numeric evidence for them.
+    """
+
+    planned_portfolio_occurrences: int
+    portfolio_scored_occurrences: int
+    portfolio_candidate_infeasible_occurrences: int
+    authenticated_mutation_observations: int
+    reflection_source_scored_occurrences: int
+    reflection_identifiable_contrasts: int
+    forecast_enabled: bool
+    planned_selector_receipts: int
+    forecast_receipts: int
+    forecast_actions: int
+    forecast_scored_actions: int
+    forecast_candidate_infeasible_actions: int
+    objective_metric_count: int
+    forecast_observations: int
+    reflection_typed_exclusions: int = 0
+
+    def __post_init__(self) -> None:
+        if type(self.forecast_enabled) is not bool:
+            raise TypeError("forecast_enabled must be an exact bool")
+        for name in (
+            "planned_portfolio_occurrences",
+            "portfolio_scored_occurrences",
+            "portfolio_candidate_infeasible_occurrences",
+            "authenticated_mutation_observations",
+            "reflection_source_scored_occurrences",
+            "reflection_identifiable_contrasts",
+            "reflection_typed_exclusions",
+            "planned_selector_receipts",
+            "forecast_receipts",
+            "forecast_actions",
+            "forecast_scored_actions",
+            "forecast_candidate_infeasible_actions",
+            "objective_metric_count",
+            "forecast_observations",
+        ):
+            value = getattr(self, name)
+            if type(value) is not int or value < 0:
+                raise ValueError(f"{name} must be a non-negative exact integer")
+        if self.objective_metric_count == 0:
+            raise ValueError("objective_metric_count must be positive")
+        if self.reflection_source_scored_occurrences > (
+            self.portfolio_scored_occurrences
+        ):
+            raise ValueError(
+                "reflection source scored occurrences exceed portfolio scores"
+            )
+
+    @property
+    def exact_portfolio_outcome_partition(self) -> bool:
+        self.__post_init__()
+        return self.planned_portfolio_occurrences == (
+            self.portfolio_scored_occurrences
+            + self.portfolio_candidate_infeasible_occurrences
+        )
+
+    @property
+    def exact_authenticated_mutation_evidence(self) -> bool:
+        self.__post_init__()
+        return (
+            self.authenticated_mutation_observations
+            == self.portfolio_scored_occurrences
+        )
+
+    @property
+    def exact_reflection_contrast_accounting(self) -> bool:
+        """Compatibility name for the exact typed reflection partition."""
+
+        return self.exact_reflection_evidence_partition
+
+    @property
+    def exact_reflection_evidence_partition(self) -> bool:
+        self.__post_init__()
+        return (
+            self.reflection_identifiable_contrasts
+            + self.reflection_typed_exclusions
+            == self.reflection_source_scored_occurrences
+        )
+
+    @property
+    def exact_forecast_feedback(self) -> bool:
+        self.__post_init__()
+        if not self.forecast_enabled:
+            return (
+                self.planned_selector_receipts == 0
+                and self.forecast_receipts == 0
+                and self.forecast_actions == 0
+                and self.forecast_scored_actions == 0
+                and self.forecast_candidate_infeasible_actions == 0
+                and self.forecast_observations == 0
+            )
+        return (
+            self.forecast_receipts == self.planned_selector_receipts
+            and self.forecast_actions == self.planned_portfolio_occurrences
+            and self.forecast_scored_actions == self.portfolio_scored_occurrences
+            and self.forecast_candidate_infeasible_actions
+            == self.portfolio_candidate_infeasible_occurrences
+            and self.forecast_observations
+            == self.portfolio_scored_occurrences * self.objective_metric_count
+        )
+
+    @property
+    def all_exact(self) -> bool:
+        self.__post_init__()
+        return (
+            self.exact_portfolio_outcome_partition
+            and self.exact_authenticated_mutation_evidence
+            and self.exact_reflection_evidence_partition
+            and self.exact_forecast_feedback
+        )
+
+    def to_record(self) -> dict[str, object]:
+        self.__post_init__()
+        return {
+            "schema_version": 2,
+            "planned_portfolio_occurrences": self.planned_portfolio_occurrences,
+            "portfolio_scored_occurrences": self.portfolio_scored_occurrences,
+            "portfolio_candidate_infeasible_occurrences": (
+                self.portfolio_candidate_infeasible_occurrences
+            ),
+            "authenticated_mutation_observations": (
+                self.authenticated_mutation_observations
+            ),
+            "reflection_source_scored_occurrences": (
+                self.reflection_source_scored_occurrences
+            ),
+            "reflection_identifiable_contrasts": (
+                self.reflection_identifiable_contrasts
+            ),
+            "reflection_typed_exclusions": self.reflection_typed_exclusions,
+            "forecast_enabled": self.forecast_enabled,
+            "planned_selector_receipts": self.planned_selector_receipts,
+            "forecast_receipts": self.forecast_receipts,
+            "forecast_actions": self.forecast_actions,
+            "forecast_scored_actions": self.forecast_scored_actions,
+            "forecast_candidate_infeasible_actions": (
+                self.forecast_candidate_infeasible_actions
+            ),
+            "objective_metric_count": self.objective_metric_count,
+            "forecast_observations": self.forecast_observations,
+            "gates": {
+                "exact_portfolio_outcome_partition": (
+                    self.exact_portfolio_outcome_partition
+                ),
+                "exact_authenticated_mutation_evidence": (
+                    self.exact_authenticated_mutation_evidence
+                ),
+                "exact_reflection_contrast_accounting": (
+                    self.exact_reflection_contrast_accounting
+                ),
+                "exact_reflection_evidence_partition": (
+                    self.exact_reflection_evidence_partition
+                ),
+                "exact_forecast_feedback": self.exact_forecast_feedback,
+            },
+            "all_exact": self.all_exact,
+        }
+
+
+__all__ = [
+    "CampaignEvaluationAccounting",
+    "CampaignPortfolioEvidenceAccounting",
+]

agent_evolve/application/evaluation_cache.py

@@ -0,0 +1,199 @@
+"""Cancellation-safe single-flight caching for expensive async evaluations.
+
+The caller owns canonical configuration hashing.  This module never serializes or
+normalizes configurations, so cache identity cannot silently lose type or value
+information.  Every normally returned value is cacheable, including a domain's
+ordinary invalid-result value; raised exceptions and cancellations are not.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections import OrderedDict
+from dataclasses import dataclass
+from enum import Enum
+from typing import Awaitable, Callable, Generic, Optional, TypeVar
+
+
+ValueT = TypeVar("ValueT")
+
+
+class EvaluationCacheEventType(str, Enum):
+    """The deliberately small observable lifecycle of the cache."""
+
+    HIT = "hit"
+    MISS = "miss"
+    COALESCED = "coalesced"
+    EVICTED = "evicted"
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationCacheSnapshot:
+    """Immutable cache occupancy and cumulative request counters."""
+
+    capacity: Optional[int]
+    cached_entries: int
+    in_flight: int
+    hits: int
+    misses: int
+    coalesced: int
+    evictions: int
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationCacheTraceEvent:
+    """One ordered cache lifecycle event with its post-event snapshot."""
+
+    sequence: int
+    event_type: EvaluationCacheEventType
+    config_hash: str
+    snapshot: EvaluationCacheSnapshot
+
+
+EvaluationFactory = Callable[[], Awaitable[ValueT]]
+EvaluationCacheTraceCallback = Callable[[EvaluationCacheTraceEvent], None]
+
+
+class AsyncEvaluationCache(Generic[ValueT]):
+    """Coalesce identical evaluations and optionally retain results in an LRU.
+
+    ``get_or_evaluate`` shields shared work from each individual awaiter.  Thus,
+    cancelling one awaiter does not cancel an evaluation still needed by another
+    awaiter (or prevent its result from being cached).  If the evaluation itself
+    raises or is cancelled, its in-flight entry is removed and a later request is
+    a fresh miss.
+
+    Trace callbacks are synchronous observation hooks.  They run in serialized
+    event order, and callback failures are isolated from cache semantics.
+    """
+
+    def __init__(
+        self,
+        *,
+        capacity: Optional[int] = None,
+        trace_callback: Optional[EvaluationCacheTraceCallback] = None,
+    ) -> None:
+        if capacity is not None and (type(capacity) is not int or capacity < 1):
+            raise ValueError("capacity must be None or a positive integer")
+        if trace_callback is not None and not callable(trace_callback):
+            raise TypeError("trace_callback must be callable or None")
+
+        self._capacity = capacity
+        self._trace_callback = trace_callback
+        self._values: "OrderedDict[str, ValueT]" = OrderedDict()
+        self._in_flight: dict[str, "asyncio.Task[ValueT]"] = {}
+        self._lock = asyncio.Lock()
+
+        self._hits = 0
+        self._misses = 0
+        self._coalesced = 0
+        self._evictions = 0
+        self._event_sequence = 0
+
+    async def get_or_evaluate(
+        self,
+        config_hash: str,
+        evaluate: EvaluationFactory[ValueT],
+    ) -> ValueT:
+        """Return a cached value or await one shared evaluation for ``config_hash``."""
+
+        self._validate_config_hash(config_hash)
+        if not callable(evaluate):
+            raise TypeError("evaluate must be callable")
+
+        async with self._lock:
+            if config_hash in self._values:
+                value = self._values.pop(config_hash)
+                self._values[config_hash] = value
+                self._hits += 1
+                self._emit_locked(EvaluationCacheEventType.HIT, config_hash)
+                return value
+
+            task = self._in_flight.get(config_hash)
+            if task is None:
+                task = asyncio.create_task(
+                    self._evaluate_and_store(config_hash, evaluate)
+                )
+                self._in_flight[config_hash] = task
+                self._misses += 1
+                self._emit_locked(EvaluationCacheEventType.MISS, config_hash)
+            else:
+                self._coalesced += 1
+                self._emit_locked(EvaluationCacheEventType.COALESCED, config_hash)
+
+        # An awaiter's cancellation must not propagate into shared evaluation work.
+        return await asyncio.shield(task)
+
+    async def snapshot(self) -> EvaluationCacheSnapshot:
+        """Return an atomic occupancy and counter snapshot."""
+
+        async with self._lock:
+            return self._snapshot_locked()
+
+    async def _evaluate_and_store(
+        self,
+        config_hash: str,
+        evaluate: EvaluationFactory[ValueT],
+    ) -> ValueT:
+        current_task = asyncio.current_task()
+        if current_task is None:  # pragma: no cover - guaranteed by create_task
+            raise RuntimeError("evaluation did not run inside an asyncio task")
+
+        try:
+            value = await evaluate()
+        except BaseException:
+            async with self._lock:
+                if self._in_flight.get(config_hash) is current_task:
+                    del self._in_flight[config_hash]
+            raise
+
+        async with self._lock:
+            if self._in_flight.get(config_hash) is not current_task:
+                raise RuntimeError("single-flight evaluation lost its ownership entry")
+            del self._in_flight[config_hash]
+            self._values[config_hash] = value
+            self._values.move_to_end(config_hash)
+
+            if self._capacity is not None and len(self._values) > self._capacity:
+                evicted_hash, _ = self._values.popitem(last=False)
+                self._evictions += 1
+                self._emit_locked(EvaluationCacheEventType.EVICTED, evicted_hash)
+
+        return value
+
+    @staticmethod
+    def _validate_config_hash(config_hash: str) -> None:
+        if type(config_hash) is not str or not config_hash:
+            raise ValueError("config_hash must be a nonempty string")
+
+    def _snapshot_locked(self) -> EvaluationCacheSnapshot:
+        return EvaluationCacheSnapshot(
+            capacity=self._capacity,
+            cached_entries=len(self._values),
+            in_flight=len(self._in_flight),
+            hits=self._hits,
+            misses=self._misses,
+            coalesced=self._coalesced,
+            evictions=self._evictions,
+        )
+
+    def _emit_locked(
+        self,
+        event_type: EvaluationCacheEventType,
+        config_hash: str,
+    ) -> None:
+        callback = self._trace_callback
+        if callback is None:
+            return
+        self._event_sequence += 1
+        event = EvaluationCacheTraceEvent(
+            sequence=self._event_sequence,
+            event_type=event_type,
+            config_hash=config_hash,
+            snapshot=self._snapshot_locked(),
+        )
+        try:
+            callback(event)
+        except Exception:
+            # Observation must not turn a valid evaluation into a failed one.
+            return