agentevolve-optimizer 0.5.0__py3-none-any.whl

agent_evolve/harness/__init__.py

@@ -0,0 +1,29 @@
+"""The Harness port and registry (LLM-runtime-agnostic)."""
+
+from agent_evolve.harness.base import (
+    OP_NAMES,
+    CallObserver,
+    Harness,
+    HarnessBase,
+    HarnessContext,
+    HarnessOutputError,
+    LLMConfig,
+    build_context,
+)
+from agent_evolve.harness.directives import DefaultDirectives, Directives
+from agent_evolve.harness.registry import HarnessRegistry, harness_registry
+
+__all__ = [
+    "OP_NAMES",
+    "CallObserver",
+    "DefaultDirectives",
+    "Directives",
+    "Harness",
+    "HarnessBase",
+    "HarnessContext",
+    "HarnessOutputError",
+    "HarnessRegistry",
+    "LLMConfig",
+    "build_context",
+    "harness_registry",
+]

agent_evolve/harness/base.py

@@ -0,0 +1,242 @@
+"""The Harness port: the switchable LLM seam the loop depends on.
+
+A harness turns the per-op inputs into **typed** candidates / insights. The loop
+never imports a concrete LLM runtime; it talks only to this interface. Concrete
+adapters live under ``agent_evolve.integrations``.
+
+Output typing is the headline: candidate ops return ``list[CandidateConfig]``
+(structured), insight ops return ``list[str]`` / ``str`` — no JSON-in-a-string.
+"""
+
+from __future__ import annotations
+
+import abc
+import inspect
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Protocol, Sequence, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict
+
+from agent_evolve.core.problem import ObjectiveSpec
+from agent_evolve.harness.directives import DefaultDirectives, Directives
+
+#: The fixed set of LLM operations exposed by every harness implementation.
+OP_NAMES: tuple[str, ...] = (
+    "generate_initial",
+    "regenerate",
+    "generate_offspring",
+    "regenerate_offspring",
+    "failure_insights",
+    "constraint_instruction",
+    "performance_insights",
+)
+
+
+class HarnessOutputError(RuntimeError):
+    """Raised when a harness call yields no usable candidates (triggers loop retry)."""
+
+
+@dataclass(frozen=True)
+class LLMConfig:
+    """Model selection and call parameters, identical across harnesses."""
+
+    model: str
+    temperature: Optional[float] = None
+    retries: int = 3
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class HarnessContext:
+    """Per-run context bound into the harness before any call."""
+
+    objectives: Sequence[ObjectiveSpec]
+    search_space_desc: str
+    candidate_model: Optional[type] = None
+    constraints_description: str = ""
+    #: Prompt wording; defaults to the generic backbone directives.
+    directives: Directives = field(default_factory=DefaultDirectives)
+
+
+class _AnyCandidate(BaseModel):
+    """Permissive fallback candidate schema for problems without a candidate_model."""
+
+    model_config = ConfigDict(extra="allow")
+
+
+def build_context(ctx: "HarnessContext") -> str:
+    """Compose the static per-run context block (objectives/search space + constraints).
+
+    This is shared *data* prep, not prompt wording — each harness composes its own
+    prompts. The candidate schema is conveyed by the typed output, not restated here.
+    """
+    parts = [ctx.search_space_desc]
+    if ctx.constraints_description:
+        parts.append(f"DOMAIN CONSTRAINTS:\n{ctx.constraints_description}")
+    return "\n\n".join(parts)
+
+
+@runtime_checkable
+class Harness(Protocol):
+    """The switchable LLM interface the loop depends on."""
+
+    id: str
+
+    def bind(self, ctx: HarnessContext, cfg: LLMConfig) -> None: ...
+    def generate_initial(self, n: int) -> List[Dict[str, Any]]: ...
+    def regenerate(
+        self, failed_str: str, n: int, constraint_instruction: str, performance_insights: str
+    ) -> List[Dict[str, Any]]: ...
+    def generate_offspring(
+        self, pareto_str: str, n: int, constraint_instruction: str, performance_insights: str
+    ) -> List[Dict[str, Any]]: ...
+    def regenerate_offspring(
+        self, failed_str: str, pareto_str: str, n: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> List[Dict[str, Any]]: ...
+    def failure_insights(self, failed_str: str, n_failed: int) -> List[str]: ...
+    def constraint_instruction(self, failed_str: str, previous: Optional[str] = None) -> str: ...
+    def performance_insights(
+        self, stats_str: str, pareto_str: str, previous: Optional[str] = None
+    ) -> str: ...
+
+
+CallObserver = Callable[[Dict[str, Any]], None]
+
+
+class HarnessContractError(TypeError):
+    """A harness subclass does not satisfy the operations it inherits."""
+
+
+def _positional_arity(func: Any) -> Optional[tuple]:
+    """Return ``(required, maximum_or_None)`` positional arity, ignoring ``self``.
+
+    ``None`` maximum means ``*args``, which accepts anything.
+    """
+    try:
+        params = list(inspect.signature(func).parameters.values())
+    except (TypeError, ValueError):  # pragma: no cover - builtins, C functions
+        return None
+    if params and params[0].name in ("self", "cls"):
+        params = params[1:]
+    required = 0
+    maximum: Optional[int] = 0
+    for p in params:
+        if p.kind is inspect.Parameter.VAR_POSITIONAL:
+            return (required, None)
+        if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD):
+            maximum = (maximum or 0) + 1
+            if p.default is inspect.Parameter.empty:
+                required += 1
+    return (required, maximum)
+
+
+class HarnessBase(abc.ABC):
+    """Shared lifecycle + result normalization for harness adapters.
+
+    **The operations are enforced, not merely declared.** This class used to
+    inherit ``abc.ABC`` while marking nothing abstract, so it promised a
+    contract and checked none of it: a subclass could omit an operation, or
+    define one with the wrong arity, and the only symptom was a ``TypeError``
+    from inside the optimization loop after the run had already started. A
+    shipped proposer did exactly that, three times.
+
+    ``__init_subclass__`` now checks, at class-definition time, that every
+    operation in :data:`OP_NAMES` exists and accepts the number of positional
+    arguments the loop passes it. A wrong signature is a failure to import the
+    module that declares it, which is when it is cheap to fix.
+    """
+
+    id: str = "base"
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        # An intermediate base may legitimately be partial; it declares so.
+        if getattr(cls, "__abstract_harness__", False):
+            return
+        problems: List[str] = []
+        for op in OP_NAMES:
+            impl = getattr(cls, op, None)
+            if impl is None or not callable(impl):
+                problems.append(f"{op}: not implemented")
+                continue
+            expected = _positional_arity(getattr(Harness, op, None))
+            actual = _positional_arity(impl)
+            if expected is None or actual is None:
+                continue
+            need, _ = expected
+            takes_min, takes_max = actual
+            if takes_max is not None and takes_max < need:
+                problems.append(
+                    f"{op}: the loop passes {need} positional argument(s), "
+                    f"this accepts at most {takes_max}"
+                )
+            elif takes_min > need:
+                problems.append(
+                    f"{op}: requires {takes_min} positional argument(s), "
+                    f"the loop passes {need}"
+                )
+        if problems:
+            raise HarnessContractError(
+                f"{cls.__name__} does not satisfy the Harness contract:\n  "
+                + "\n  ".join(problems)
+            )
+
+    def __init__(self) -> None:
+        self._ctx: Optional[HarnessContext] = None
+        self._cfg: Optional[LLMConfig] = None
+        self._observer: Optional[CallObserver] = None
+        self.context: str = ""
+        self.candidate_type: type = _AnyCandidate
+
+    def bind(self, ctx: HarnessContext, cfg: LLMConfig) -> None:
+        self._ctx = ctx
+        self._cfg = cfg
+        self.context = build_context(ctx)
+        self.candidate_type = ctx.candidate_model or _AnyCandidate
+        self._on_bind(ctx, cfg)
+
+    def set_call_observer(self, observer: Optional[CallObserver]) -> None:
+        self._observer = observer
+
+    @property
+    def cfg(self) -> LLMConfig:
+        if self._cfg is None:
+            raise RuntimeError("Harness used before bind()")
+        return self._cfg
+
+    @property
+    def directives(self) -> Directives:
+        if self._ctx is None:
+            raise RuntimeError("Harness used before bind()")
+        return self._ctx.directives
+
+    # -- subclass hook ---------------------------------------------------
+    def _on_bind(self, ctx: HarnessContext, cfg: LLMConfig) -> None:
+        """Optional adapter-specific setup."""
+
+    # -- normalization helpers ------------------------------------------
+    def _observe(self, op: str) -> None:
+        if self._observer is not None:
+            self._observer({"kind": "llm_call", "op": op, "harness": self.id,
+                            "model": self.cfg.model})
+
+    @staticmethod
+    def _to_dicts(objs: Any, op: str) -> List[Dict[str, Any]]:
+        if not isinstance(objs, (list, tuple)) or not objs:
+            raise HarnessOutputError(f"{op}: model returned no candidates")
+        out: List[Dict[str, Any]] = []
+        for o in objs:
+            if isinstance(o, BaseModel):
+                out.append(o.model_dump())
+            elif isinstance(o, dict):
+                out.append(o)
+            else:
+                raise HarnessOutputError(f"{op}: unexpected candidate type {type(o).__name__}")
+        return out
+
+    @staticmethod
+    def _to_str_list(objs: Any) -> List[str]:
+        if isinstance(objs, (list, tuple)):
+            return [str(x) for x in objs]
+        return [str(objs)]

agent_evolve/harness/directives.py

@@ -0,0 +1,163 @@
+"""The Directives port: problem-supplied prompt text.
+
+A ``Directives`` provider supplies *what to say* to the LLM while harness
+adapters supply *how to run it*. The backbone ships
+:class:`DefaultDirectives` (generic wording) so examples and new problems work
+out of the box; a problem can expose its own ``directives`` attribute to fully
+own its prompts.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+#: Chain-of-thought instruction, paired with a Proposal(thought_process,
+#: candidates) output type so the model reasons before committing.
+#:
+#: This wording used to instruct the model to "split its factors across the
+#: levels so every per-dimension product is EXACTLY correct" -- the constraint
+#: of the one problem the backbone was first written against, shipped as if it
+#: were generic. Every other problem received an instruction about a structure
+#: it does not have. ``DefaultDirectives`` is the *problem-agnostic* wording by
+#: definition; anything specific to one search space belongs in that problem's
+#: own ``Directives``, which is exactly what the port is for.
+_COT = (
+    " First, in thought_process, reason step by step as a mental draft: state what the "
+    "measurements so far imply about which parts of the configuration matter, decide what "
+    "to change and by how much, and check each proposal against the stated constraints and "
+    "the declared schema. Then fill candidates."
+)
+
+
+@runtime_checkable
+class Directives(Protocol):
+    """Prompt wording for each operation."""
+
+    def compose_initial(self, context: str, n_candidates: int) -> str: ...
+    def compose_regenerate(
+        self, context: str, failed_str: str, n_candidates: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> str: ...
+    def compose_offspring(
+        self, context: str, pareto_str: str, n_candidates: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> str: ...
+    def compose_regenerate_offspring(
+        self, context: str, failed_str: str, pareto_str: str, n_candidates: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> str: ...
+    def compose_failure_insights(self, context: str, failed_str: str, n_failed: int) -> str: ...
+    def compose_constraint_instruction(self, context: str, failed_str: str, previous: str = "") -> str: ...
+    def compose_performance_insights(
+        self, context: str, stats_str: str, pareto_str: str, previous: str = ""
+    ) -> str: ...
+
+
+class DefaultDirectives:
+    """Generic, problem-agnostic prompt wording.
+
+    Problem specifics (search space, constraints, examples) reach the prompt
+    through ``context`` (built from the problem's hooks), so this generic wording
+    works for any problem; a problem may still supply its own ``Directives``.
+    """
+
+    def compose_initial(self, context: str, n_candidates: int) -> str:
+        return (
+            "You are an optimization expert generating candidates for a multi-objective problem.\n\n"
+            f"{context}\n\n"
+            f"Propose exactly {n_candidates} diverse, valid candidates that explore different "
+            "regions of the search space and trade off the objectives."
+            + _COT
+        )
+
+    def compose_regenerate(
+        self, context: str, failed_str: str, n_candidates: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> str:
+        return (
+            "You are an optimization expert. Previous candidates failed validation; learn from "
+            "them and propose better ones.\n\n"
+            f"{context}\n\n"
+            f"FAILED CANDIDATES AND THEIR ISSUES:\n{failed_str}\n\n"
+            f"CONSTRAINT COMPLIANCE INSTRUCTIONS:\n{constraint_instruction or 'None yet.'}\n\n"
+            f"PERFORMANCE INSIGHTS:\n{performance_insights or 'None yet.'}\n\n"
+            f"Propose exactly {n_candidates} NEW valid candidates that fix the failures and follow "
+            "the constraint instructions."
+            + _COT
+        )
+
+    def compose_offspring(
+        self, context: str, pareto_str: str, n_candidates: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> str:
+        return (
+            "You are an optimization expert. Breed offspring from high-quality Pareto-optimal "
+            "configurations.\n\n"
+            f"{context}\n\n"
+            f"PARETO-OPTIMAL CONFIGURATIONS (best so far):\n{pareto_str}\n\n"
+            f"CONSTRAINT COMPLIANCE INSTRUCTIONS:\n{constraint_instruction or 'Follow standard constraints.'}\n\n"
+            f"PERFORMANCE INSIGHTS:\n{performance_insights or 'Analyze the Pareto set for patterns.'}\n\n"
+            f"Propose exactly {n_candidates} NEW valid candidates that build on the Pareto patterns, "
+            "explore new trade-offs, and try to improve on existing solutions."
+            + _COT
+        )
+
+    def compose_regenerate_offspring(
+        self, context: str, failed_str: str, pareto_str: str, n_candidates: int,
+        constraint_instruction: str, performance_insights: str,
+    ) -> str:
+        return (
+            "You are an optimization expert. Some offspring failed; fix them while staying close to "
+            "the known-valid Pareto set.\n\n"
+            f"{context}\n\n"
+            f"PARETO-OPTIMAL CONFIGURATIONS (known valid, high quality):\n{pareto_str}\n\n"
+            f"FAILED OFFSPRING AND THEIR ISSUES:\n{failed_str}\n\n"
+            f"CONSTRAINT COMPLIANCE INSTRUCTIONS:\n{constraint_instruction or 'Follow the Pareto patterns.'}\n\n"
+            f"PERFORMANCE INSIGHTS:\n{performance_insights or ''}\n\n"
+            f"Propose exactly {n_candidates} NEW valid candidates that fix the failures and try to improve."
+            + _COT
+        )
+
+    def compose_failure_insights(self, context: str, failed_str: str, n_failed: int) -> str:
+        return (
+            "You are an optimization expert. Analyze why each candidate failed and give one "
+            "specific, actionable fix per candidate.\n\n"
+            f"{context}\n\n"
+            f"FAILED CANDIDATES:\n{failed_str}\n\n"
+            f"Provide exactly {n_failed} insight strings, one per failed candidate, in order."
+        )
+
+    def compose_constraint_instruction(self, context: str, failed_str: str, previous: str = "") -> str:
+        if previous:
+            return (
+                "You are an optimization expert. Update the constraint-compliance guide using new failures.\n\n"
+                f"PREVIOUS GUIDE:\n{previous}\n\n"
+                f"NEW FAILED CANDIDATES AND INSIGHTS:\n{failed_str}\n\n"
+                "Produce an updated, comprehensive constraint-compliance guide."
+            )
+        return (
+            "You are an optimization expert. From these failures, write a consolidated "
+            "constraint-compliance guide future candidates should follow.\n\n"
+            f"{context}\n\n"
+            f"FAILED CANDIDATES AND INSIGHTS:\n{failed_str}\n\n"
+            "Produce a clear, actionable constraint-compliance guide."
+        )
+
+    def compose_performance_insights(
+        self, context: str, stats_str: str, pareto_str: str, previous: str = ""
+    ) -> str:
+        if previous:
+            return (
+                "You are an optimization expert. Update the performance insights with new results.\n\n"
+                f"PREVIOUS INSIGHTS:\n{previous}\n\n"
+                f"CURRENT PERFORMANCE STATISTICS:\n{stats_str}\n\n"
+                f"CURRENT TOP PARETO CONFIGURATIONS:\n{pareto_str}\n\n"
+                "Produce updated performance insights that capture new patterns."
+            )
+        return (
+            "You are an optimization expert. Analyze the performance patterns to guide better candidates.\n\n"
+            f"{context}\n\n"
+            f"PERFORMANCE STATISTICS:\n{stats_str}\n\n"
+            f"TOP PARETO CONFIGURATIONS:\n{pareto_str}\n\n"
+            "Explain what makes configurations perform well, the trade-offs, and how to improve."
+        )