agentevolve-optimizer 0.5.0__py3-none-any.whl

agent_evolve/core/formatting.py

@@ -0,0 +1,286 @@
+"""Pure formatting/parsing helpers shared by the loop and the LLM prompts.
+
+Everything here is a pure function of its arguments (no I/O, no globals).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+
+from pydantic import BaseModel
+
+from agent_evolve.core.problem import ObjectiveSpec
+from agent_evolve.core.results import Candidate, objective_value
+
+
+@dataclass
+class CandidateResult:
+    """Intermediate evaluation record (richer than the public :class:`Candidate`)."""
+
+    configuration: Dict[str, Any]
+    objectives: Dict[str, float]
+    is_valid: bool
+    error_message: Optional[str] = None
+    failure_phase: Optional[str] = None
+    #: True iff ``Problem.evaluate`` was actually invoked for this candidate.
+    evaluation_attempted: bool = False
+    insight: str = ""
+    #: Original element from the LLM ``candidates`` list (before/while parsing).
+    raw_llm_element: Optional[Any] = None
+
+
+# ------------------------------------------------------------------
+# Prettifiers (consumed by LLM prompts)
+# ------------------------------------------------------------------
+
+def prettify_configuration(config: Dict[str, Any], indent: int = 2) -> str:
+    return json.dumps(config, indent=indent, sort_keys=True, default=str)
+
+
+def dump_raw_llm_element(obj: Any, *, max_len: int = 12_000) -> str:
+    """Serialize a raw LLM list element for logs and failure prompts."""
+    if obj is None:
+        return "(none)"
+    try:
+        s = json.dumps(obj, indent=2, default=str, ensure_ascii=False)
+    except (TypeError, ValueError):
+        s = repr(obj)
+    if len(s) > max_len:
+        return s[:max_len] + f"\n... [truncated, {len(s)} chars total]"
+    return s
+
+
+def prettify_objectives(objectives: Sequence[ObjectiveSpec]) -> str:
+    lines = ["OBJECTIVES:", "=" * 60]
+    for spec in objectives:
+        desc = "higher is better" if spec.goal == "max" else "lower is better"
+        lines.append(f"  - {spec.name}: {desc}")
+    return "\n".join(lines)
+
+
+def prettify_results(
+    results: Sequence[CandidateResult],
+    objectives: Sequence[ObjectiveSpec],
+    render: Optional[Callable[[Dict[str, Any]], str]] = None,
+) -> str:
+    """Format candidate results for LLM prompts.
+
+    ``render`` optionally produces a compact one-line view of each configuration
+    (e.g. a problem-specific summary); when absent, pretty JSON is used.
+    """
+    lines: List[str] = []
+    for i, r in enumerate(results, 1):
+        lines.append(f"--- Candidate {i} ---")
+        config_str = render(r.configuration) if render is not None else prettify_configuration(r.configuration)
+        lines.append(f"Configuration: {config_str}")
+        if getattr(r, "raw_llm_element", None) is not None:
+            lines.append(
+                "Raw LLM element (exact item from the model's candidates list): "
+                + dump_raw_llm_element(r.raw_llm_element)
+            )
+        if r.is_valid:
+            parts = []
+            for spec in objectives:
+                val = objective_value(r.objectives, spec.name)
+                arrow = "\u2191" if spec.goal == "max" else "\u2193"
+                parts.append(f"{spec.name}={val:.4f}{arrow}")
+            lines.append(f"Objectives: {', '.join(parts)}")
+        else:
+            lines.append("Status: INVALID")
+            if r.failure_phase:
+                lines.append(f"Failure Phase: {r.failure_phase}")
+            if r.error_message:
+                lines.append(f"Error: {r.error_message}")
+        if r.insight:
+            lines.append(f"Insight: {r.insight}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+# ------------------------------------------------------------------
+# Search-space description for LLM context
+# ------------------------------------------------------------------
+
+def format_search_space_description(
+    objectives: Sequence[ObjectiveSpec],
+    *,
+    config_schema: Optional[Dict[str, Any]] = None,
+    example_config: Optional[Dict[str, Any]] = None,
+    constraints: Optional[str] = None,
+    problem_description: Optional[str] = None,
+) -> str:
+    lines: List[str] = []
+    lines.append("=" * 70)
+    lines.append("MULTI-OBJECTIVE OPTIMIZATION PROBLEM")
+    lines.append("=" * 70)
+    lines.append("")
+    lines.append("OBJECTIVES:")
+    for spec in objectives:
+        desc = "MAXIMIZE (higher is better)" if spec.goal == "max" else "MINIMIZE (lower is better)"
+        lines.append(f"  \u2022 {spec.name}: {desc}")
+    lines.append("")
+
+    if problem_description:
+        lines.append("PROBLEM DESCRIPTION:")
+        lines.append(problem_description)
+        lines.append("")
+
+    if config_schema:
+        lines.append("CONFIGURATION SCHEMA:")
+        lines.append(prettify_configuration(config_schema))
+        lines.append("")
+
+    if example_config:
+        lines.append("EXAMPLE CONFIGURATION:")
+        lines.append(prettify_configuration(example_config))
+        lines.append("")
+
+    if constraints:
+        lines.append("CONSTRAINTS:")
+        lines.append(constraints)
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+# ------------------------------------------------------------------
+# Conversions between CandidateResult and the public Candidate
+# ------------------------------------------------------------------
+
+def result_to_candidate(
+    result: CandidateResult,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Candidate[Dict[str, Any]]:
+    return Candidate(
+        configuration=result.configuration,
+        objectives=result.objectives,
+        metadata=metadata or {"is_pareto": False},
+    )
+
+
+def candidate_to_result(candidate: Candidate[Dict[str, Any]]) -> CandidateResult:
+    return CandidateResult(
+        configuration=candidate.configuration,
+        objectives=candidate.objectives,
+        is_valid=True,
+        evaluation_attempted=True,
+    )
+
+
+# ------------------------------------------------------------------
+# Parse LLM candidate output
+# ------------------------------------------------------------------
+
+def parse_llm_json_array(s: str) -> List[Any]:
+    """Parse a JSON array (or single object) from an LLM ``str`` field.
+
+    The single-JSON-string output contract avoids the ``[{}, {}, ...]`` degradation
+    that ``list[dict]`` structured outputs are prone to. Strips optional markdown fences.
+    """
+    s = (s or "").strip()
+    if not s:
+        raise ValueError("Empty candidates JSON string")
+    if s.startswith("```"):
+        s = re.sub(r"^```(?:json)?\s*", "", s, flags=re.IGNORECASE)
+        s = re.sub(r"\s*```\s*$", "", s)
+    data = json.loads(s)
+    if isinstance(data, dict):
+        inner = data.get("candidates")
+        if isinstance(inner, list):
+            return inner
+        return [data]
+    if isinstance(data, list):
+        return data
+    raise ValueError(f"Candidates JSON must be an array or object, got {type(data).__name__}")
+
+
+def parse_candidates(
+    candidates: Any,
+    expected_count: int,
+    log_fn: Callable[[str], None] = lambda m: None,
+) -> Tuple[List[Dict[str, Any]], List[Any]]:
+    """Normalise LLM output into configuration dicts.
+
+    Returns ``(parsed_configs, raw_elements)`` with one raw element per input list
+    item (same order), so failures can show what the model actually returned even
+    when the parsed dict is ``{}`` or wrong.
+    """
+    if isinstance(candidates, str):
+        try:
+            candidates = parse_llm_json_array(candidates)
+        except Exception as exc:
+            log_fn(f"Warning: could not parse candidates as JSON array string: {exc}")
+            return [], []
+
+    if isinstance(candidates, dict):
+        inner = candidates.get("candidates")
+        if isinstance(inner, list):
+            candidates = inner
+        else:
+            log_fn(
+                f"Warning: LLM returned a dict without a 'candidates' list "
+                f"(keys: {list(candidates.keys())})."
+            )
+            return [], []
+
+    if not isinstance(candidates, list):
+        log_fn(f"Warning: LLM returned non-list candidates: {type(candidates)}")
+        return [], []
+
+    parsed: List[Dict[str, Any]] = []
+    raw_elements: List[Any] = []
+    for c in candidates:
+        raw_elements.append(c)
+        if isinstance(c, BaseModel):
+            parsed.append(c.model_dump())
+        elif isinstance(c, dict):
+            parsed.append(c)
+        elif isinstance(c, str):
+            try:
+                parsed.append(json.loads(c))
+            except Exception:
+                log_fn(f"Warning: Could not parse candidate string: {c[:100]}")
+                parsed.append({})
+        else:
+            log_fn(
+                f"Warning: candidate element has unexpected type {type(c).__name__}"
+            )
+            parsed.append({})
+
+    if len(parsed) != expected_count:
+        log_fn(f"Warning: Expected {expected_count} candidates, got {len(parsed)}")
+    return parsed, raw_elements
+
+
+def result_to_json(result: Any) -> str:
+    """One machine-readable document for a ``SearchResult``.
+
+    Blocks that were never populated serialize as ``null`` rather than being
+    omitted, so a reader can tell "measured zero" (a present block with zero
+    counts) from "nobody looked" (null). Values outside JSON's vocabulary fall
+    back to ``str`` -- a printable document beats a crash on an exotic locus
+    type, and the exact values live in the caller's hands anyway.
+    """
+    import dataclasses
+
+    def _candidate(c: Any) -> Dict[str, Any]:
+        return {"configuration": c.configuration, "objectives": c.objectives}
+
+    payload = {
+        "best": _candidate(result.best),
+        "pareto_front": [_candidate(c) for c in result.pareto_front],
+        "evaluations": result.evaluations,
+        "history": result.history,
+        "provider_usage": (
+            dataclasses.asdict(result.provider_usage)
+            if result.provider_usage is not None else None
+        ),
+        "telemetry": (
+            dataclasses.asdict(result.telemetry)
+            if result.telemetry is not None else None
+        ),
+    }
+    return json.dumps(payload, sort_keys=True, default=str)

agent_evolve/core/optimization_semantics.py

@@ -0,0 +1,324 @@
+"""Immutable, prompt-safe semantics for optimization metrics and ordering.
+
+The application core must not guess what a benchmark metric means.  This
+module gives benchmark adapters a small inverted API for publishing that
+meaning once, binding it to the objective and outcome-relation identities,
+and rendering the same canonical record into every agentic prompt.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Sequence
+
+from agent_evolve.core.problem import ObjectiveSpec, validate_objective_specs
+from agent_evolve.domain.patch import require_sha256
+
+
+_TOKEN = re.compile(r"^[a-z][a-z0-9_.-]{0,95}$")
+_SEMANTICS_HASH_DOMAIN = b"agent-evolve:optimization-semantics:v1\x00"
+
+
+class MetricRole(str, Enum):
+    """Closed role vocabulary for values exposed to an optimizer."""
+
+    OBJECTIVE = "objective"
+    VIOLATION = "violation"
+    CONSTRAINT = "constraint"
+    DIAGNOSTIC = "diagnostic"
+
+
+class MetricSense(str, Enum):
+    """How better values of a metric are interpreted."""
+
+    MINIMIZE = "minimize"
+    MAXIMIZE = "maximize"
+    TARGET = "target"
+    SATISFY_BOUNDS = "satisfy_bounds"
+    INFORMATIONAL = "informational"
+
+
+class OutcomeOrderingKind(str, Enum):
+    """High-level structure of the benchmark's outcome comparison."""
+
+    LEXICOGRAPHIC = "lexicographic"
+    PARETO = "pareto"
+    SCALAR = "scalar"
+    CUSTOM = "custom"
+
+
+def _nonempty(value: object, name: str) -> str:
+    if type(value) is not str or not value.strip():
+        raise ValueError(f"{name} must be a non-empty exact string")
+    return value
+
+
+def _token(value: object, name: str) -> str:
+    text = _nonempty(value, name)
+    if _TOKEN.fullmatch(text) is None:
+        raise ValueError(f"{name} must use the closed token grammar")
+    return text
+
+
+def _finite_optional(value: object, name: str) -> float | None:
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise TypeError(f"{name} must be a finite number or None")
+    number = float(value)
+    if not math.isfinite(number):
+        raise ValueError(f"{name} must be finite")
+    return number
+
+
+@dataclass(frozen=True, slots=True)
+class MetricSemantics:
+    """Exact human-facing meaning of one objective/evidence metric."""
+
+    metric_id: str
+    name: str
+    role: MetricRole
+    sense: MetricSense
+    definition: str
+    aggregation: str
+    witness_interpretation: str
+    reference_target: float | None = None
+    bounds: tuple[float | None, float | None] | None = None
+    tolerance: float | None = None
+
+    def __post_init__(self) -> None:
+        _token(self.name, "metric name")
+        if type(self.role) is not MetricRole:
+            raise TypeError("role must be an exact MetricRole")
+        if type(self.sense) is not MetricSense:
+            raise TypeError("sense must be an exact MetricSense")
+        expected_prefix = f"{self.role.value}:"
+        if (
+            type(self.metric_id) is not str
+            or not self.metric_id.startswith(expected_prefix)
+            or _TOKEN.fullmatch(self.metric_id[len(expected_prefix) :]) is None
+        ):
+            raise ValueError(
+                "metric_id must be '<role>:<closed-token-name>' and match role"
+            )
+        _nonempty(self.definition, "metric definition")
+        _nonempty(self.aggregation, "metric aggregation")
+        _nonempty(self.witness_interpretation, "witness_interpretation")
+        target = _finite_optional(self.reference_target, "reference_target")
+        tolerance = _finite_optional(self.tolerance, "tolerance")
+        if tolerance is not None and tolerance < 0:
+            raise ValueError("tolerance must be non-negative")
+        if self.sense is MetricSense.TARGET and target is None:
+            raise ValueError("target-sense metrics require reference_target")
+        if self.bounds is not None:
+            if type(self.bounds) is not tuple or len(self.bounds) != 2:
+                raise TypeError("bounds must be an exact (lower, upper) tuple")
+            lower = _finite_optional(self.bounds[0], "bounds lower")
+            upper = _finite_optional(self.bounds[1], "bounds upper")
+            if lower is None and upper is None:
+                raise ValueError("bounds must publish at least one endpoint")
+            if lower is not None and upper is not None and lower > upper:
+                raise ValueError("bounds lower endpoint exceeds upper endpoint")
+        elif self.sense is MetricSense.SATISFY_BOUNDS:
+            raise ValueError("satisfy-bounds metrics require bounds")
+
+    def to_record(self) -> dict[str, object]:
+        return {
+            "metric_id": self.metric_id,
+            "name": self.name,
+            "role": self.role.value,
+            "sense": self.sense.value,
+            "definition": self.definition,
+            "aggregation": self.aggregation,
+            "reference_target": self.reference_target,
+            "bounds": None if self.bounds is None else list(self.bounds),
+            "tolerance": self.tolerance,
+            "witness_interpretation": self.witness_interpretation,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class OutcomeOrderingSemantics:
+    """Human-readable ordering bound to the executable relation policy."""
+
+    kind: OutcomeOrderingKind
+    metric_priority: tuple[str, ...]
+    description: str
+    equivalence: str
+    policy_id: str
+    policy_version: int
+    definition_sha256: str
+
+    def __post_init__(self) -> None:
+        if type(self.kind) is not OutcomeOrderingKind:
+            raise TypeError("kind must be an exact OutcomeOrderingKind")
+        if type(self.metric_priority) is not tuple or not self.metric_priority:
+            raise ValueError("metric_priority must be a non-empty exact tuple")
+        if any(type(value) is not str or not value for value in self.metric_priority):
+            raise TypeError("metric_priority entries must be non-empty strings")
+        if len(set(self.metric_priority)) != len(self.metric_priority):
+            raise ValueError("metric_priority must not contain duplicates")
+        _nonempty(self.description, "outcome ordering description")
+        _nonempty(self.equivalence, "outcome equivalence description")
+        _token(self.policy_id, "outcome policy_id")
+        if type(self.policy_version) is not int or self.policy_version <= 0:
+            raise ValueError("outcome policy_version must be a positive exact integer")
+        require_sha256(self.definition_sha256, "outcome definition_sha256")
+
+    @property
+    def relation_identity(self) -> tuple[str, int, str]:
+        return self.policy_id, self.policy_version, self.definition_sha256
+
+    def to_record(self) -> dict[str, object]:
+        return {
+            "kind": self.kind.value,
+            "metric_priority": list(self.metric_priority),
+            "description": self.description,
+            "equivalence": self.equivalence,
+            "relation_policy": {
+                "policy_id": self.policy_id,
+                "policy_version": self.policy_version,
+                "definition_sha256": self.definition_sha256,
+            },
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class OptimizationSemantics:
+    """Versioned semantic contract supplied by a benchmark adapter."""
+
+    semantics_id: str
+    semantics_version: int
+    metrics: tuple[MetricSemantics, ...]
+    outcome_ordering: OutcomeOrderingSemantics
+    definition_sha256: str = field(init=False)
+
+    def __post_init__(self) -> None:
+        _token(self.semantics_id, "semantics_id")
+        if type(self.semantics_version) is not int or self.semantics_version <= 0:
+            raise ValueError("semantics_version must be a positive exact integer")
+        if type(self.metrics) is not tuple or not self.metrics:
+            raise ValueError("metrics must be a non-empty exact tuple")
+        for metric in self.metrics:
+            if type(metric) is not MetricSemantics:
+                raise TypeError("metrics must contain exact MetricSemantics values")
+            MetricSemantics.__post_init__(metric)
+        metric_ids = tuple(metric.metric_id for metric in self.metrics)
+        if len(set(metric_ids)) != len(metric_ids):
+            raise ValueError("metric IDs must be unique")
+        if type(self.outcome_ordering) is not OutcomeOrderingSemantics:
+            raise TypeError(
+                "outcome_ordering must be an exact OutcomeOrderingSemantics"
+            )
+        OutcomeOrderingSemantics.__post_init__(self.outcome_ordering)
+        missing = set(self.outcome_ordering.metric_priority) - set(metric_ids)
+        if missing:
+            raise ValueError(
+                "outcome metric_priority references unknown metrics: "
+                + ", ".join(sorted(missing))
+            )
+        encoded = json.dumps(
+            self._definition_record(),
+            allow_nan=False,
+            ensure_ascii=True,
+            separators=(",", ":"),
+            sort_keys=True,
+        ).encode("ascii")
+        object.__setattr__(
+            self,
+            "definition_sha256",
+            hashlib.sha256(_SEMANTICS_HASH_DOMAIN + encoded).hexdigest(),
+        )
+
+    def _definition_record(self) -> dict[str, object]:
+        return {
+            "schema_version": 1,
+            "semantics_id": self.semantics_id,
+            "semantics_version": self.semantics_version,
+            "metrics": [metric.to_record() for metric in self.metrics],
+            "outcome_ordering": self.outcome_ordering.to_record(),
+        }
+
+    @property
+    def identity(self) -> tuple[str, int, str]:
+        return self.semantics_id, self.semantics_version, self.definition_sha256
+
+    def to_record(self) -> dict[str, object]:
+        return {
+            **self._definition_record(),
+            "definition_sha256": self.definition_sha256,
+        }
+
+    def validate_binding(
+        self,
+        objectives: Sequence[ObjectiveSpec],
+        outcome_relation_identity: tuple[str, int, str],
+    ) -> None:
+        """Bind published prose to executable objective and relation semantics."""
+
+        validate_objective_specs(objectives)
+        if self.outcome_ordering.relation_identity != outcome_relation_identity:
+            raise ValueError(
+                "optimization semantics outcome ordering differs from the "
+                "executable outcome relation"
+            )
+        objective_metrics = {
+            metric.name: metric
+            for metric in self.metrics
+            if metric.role is MetricRole.OBJECTIVE
+        }
+        expected_names = {objective.name for objective in objectives}
+        if set(objective_metrics) != expected_names:
+            raise ValueError(
+                "optimization semantics objective metrics differ from declared "
+                "problem objectives"
+            )
+        for objective in objectives:
+            expected_sense = (
+                MetricSense.MINIMIZE
+                if objective.goal == "min"
+                else MetricSense.MAXIMIZE
+            )
+            if objective_metrics[objective.name].sense is not expected_sense:
+                raise ValueError(
+                    f"optimization semantics sense differs for {objective.name!r}"
+                )
+
+
+def render_optimization_semantics(semantics: OptimizationSemantics) -> str:
+    """Return the canonical prompt block shared by proposal and reflection."""
+
+    if type(semantics) is not OptimizationSemantics:
+        raise TypeError("semantics must be an exact OptimizationSemantics")
+    OptimizationSemantics.__post_init__(semantics)
+    payload = json.dumps(
+        semantics.to_record(),
+        allow_nan=False,
+        ensure_ascii=True,
+        separators=(",", ":"),
+        sort_keys=True,
+    )
+    return "\n".join(
+        (
+            "OPTIMIZATION SEMANTICS (VERSIONED, AUTHORITATIVE)",
+            "Use these exact metric definitions, witness signs, and outcome "
+            "ordering; do not infer semantics from metric names.",
+            payload,
+        )
+    )
+
+
+__all__ = [
+    "MetricRole",
+    "MetricSemantics",
+    "MetricSense",
+    "OptimizationSemantics",
+    "OutcomeOrderingKind",
+    "OutcomeOrderingSemantics",
+    "render_optimization_semantics",
+]