synth-optimizers 0.1.0__py3-none-any.whl

prompt_opt/gepa_ai_compat.py

@@ -0,0 +1,225 @@
+"""GEPA-compatible local optimizer facade backed by the local offline SDK."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from prompt_opt.adapters.synth_offline import SynthOfflineLearningAdapter
+from prompt_opt.sdk.optimization.internal.prompt_learning import PromptLearningJob
+
+try:
+    from synth_ai.gepa.core.result import GEPAResult as SynthGEPAResult
+except Exception:  # pragma: no cover - allows standalone use
+    SynthGEPAResult = None
+
+
+class LocalGEPAAdapterProtocol(Protocol):
+    """Protocol compatible with GEPA adapter-style calls."""
+
+    def evaluate(
+        self,
+        batch: list[Mapping[str, Any]],
+        candidate: dict[str, str],
+        capture_traces: bool = False,
+    ) -> Any:
+        ...
+
+    def make_reflective_dataset(
+        self,
+        candidate: dict[str, str],
+        eval_batch: Any,
+        components_to_update: list[str],
+    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
+        ...
+
+
+@dataclass(frozen=True)
+class LocalGEPAResult:
+    """Minimal GEPA-style result when synth_ai is unavailable."""
+
+    candidates: list[dict[str, str]]
+    parents: list[list[int | None]]
+    val_aggregate_scores: list[float]
+    val_subscores: list[dict[int, float]]
+    per_val_instance_best_candidates: dict[int, set[int]]
+    discovery_eval_counts: list[int]
+    total_metric_calls: int
+
+    @property
+    def best_idx(self) -> int:
+        return max(range(len(self.val_aggregate_scores)), key=self.val_aggregate_scores.__getitem__)
+
+    @property
+    def best_candidate(self) -> dict[str, str]:
+        return self.candidates[self.best_idx]
+
+
+def optimize(
+    seed_candidate: dict[str, str],
+    trainset: list[Mapping[str, Any]],
+    valset: list[Mapping[str, Any]] | None = None,
+    adapter: LocalGEPAAdapterProtocol | None = None,
+    task_lm: str | Any | None = None,
+    evaluator: Any | None = None,
+    reflection_lm: str | Any | None = None,
+    max_metric_calls: int | None = None,
+    stop_callbacks: Any | None = None,
+    **_: Any,
+) -> Any:
+    """Local GEPA-compatible optimize function.
+
+    This function is shaped for `gepa-ai` compatibility and can be used as a
+    drop-in local replacement where a simple adapter-based optimization loop is
+    sufficient.
+    """
+    del task_lm, evaluator, reflection_lm, stop_callbacks
+    if not seed_candidate:
+        raise ValueError("seed_candidate must contain at least one entry.")
+    if not trainset:
+        raise ValueError("trainset must contain at least one item.")
+    if adapter is None:
+        raise ValueError("adapter is required for local/offline mode.")
+
+    eval_set: list[Mapping[str, Any]] = list(valset if valset is not None else trainset)
+    budget = max(1, int(max_metric_calls or 8))
+    stages = [
+        {
+            "id": key,
+            "name": key,
+            "messages": [{"role": "system", "pattern": value, "order": 0}],
+            "wildcards": {},
+        }
+        for key, value in seed_candidate.items()
+    ]
+    if not stages:
+        stages = [
+            {
+                "id": "default",
+                "name": "default",
+                "messages": [{"role": "system", "pattern": "You are a helpful assistant.", "order": 0}],
+                "wildcards": {},
+            }
+        ]
+
+    prompt_learning_config = {
+        "prompt_learning": {
+            "algorithm": "gepa",
+            "execution_mode": "retrieved",
+            "task_data": {
+                "train_examples": [dict(item) for item in trainset],
+                "validation_examples": [dict(item) for item in eval_set],
+            },
+            "gepa": {
+                "initial_candidate": {"stages": stages},
+                "population": {
+                    "initial_size": 1,
+                    "num_generations": 1,
+                    "children_per_generation": max(1, budget - 1),
+                },
+                "termination_conditions": {
+                    "total_rollouts": budget * max(1, len(eval_set)),
+                },
+            },
+            "local_runtime": {
+                "adapter": adapter,
+            },
+        }
+    }
+    job = PromptLearningJob.from_dict(
+        prompt_learning_config,
+        backend_url="local://prompt-opt",
+        api_key="local",
+    )
+    job.submit()
+    job.stream_until_complete(timeout=300.0, interval=0.05).to_dict()
+    candidates_page = job.list_candidates(limit=budget + 1)
+    raw_candidates = candidates_page.get("items", [])
+
+    candidates: list[dict[str, str]] = []
+    parents: list[list[int | None]] = []
+    scores: list[float] = []
+    subscores: list[dict[int, float]] = []
+    eval_counts: list[int] = []
+    metric_calls_total = 0
+    candidate_index_by_id: dict[str, int] = {}
+
+    seed_eval_items = job.get_state_envelope().get("state", {}).get("seed_evals", [])
+    per_candidate_per_seed: dict[str, dict[int, float]] = {}
+    for seed_eval in seed_eval_items:
+        if not isinstance(seed_eval, dict):
+            continue
+        candidate_id = str(seed_eval.get("candidate_id", ""))
+        seed = int(seed_eval.get("seed", 0))
+        reward = float(seed_eval.get("reward", 0.0))
+        per_candidate_per_seed.setdefault(candidate_id, {})[seed] = reward
+
+    for raw_candidate in raw_candidates:
+        if not isinstance(raw_candidate, dict):
+            continue
+        stage_items = raw_candidate.get("stages") or raw_candidate.get("candidate", {}).get("stages") or []
+        candidate_map: dict[str, str] = {}
+        for stage in stage_items:
+            if not isinstance(stage, dict):
+                continue
+            stage_key = str(stage.get("id") or stage.get("name") or f"stage_{len(candidate_map)}")
+            for message in stage.get("messages", []):
+                if isinstance(message, dict) and message.get("role") == "system":
+                    text = message.get("pattern") or message.get("content")
+                    if isinstance(text, str):
+                        candidate_map[stage_key] = text
+                        break
+        if not candidate_map:
+            candidate_map = {
+                "default": str(raw_candidate.get("candidate_content", "")),
+            }
+        candidate_id = str(raw_candidate.get("candidate_id", ""))
+        candidate_index_by_id[candidate_id] = len(candidates)
+        candidates.append(candidate_map)
+        parent_id = raw_candidate.get("parent_id")
+        parents.append([candidate_index_by_id[parent_id]] if isinstance(parent_id, str) and parent_id in candidate_index_by_id else [None])
+        candidate_subscores = per_candidate_per_seed.get(candidate_id, {})
+        subscores.append(dict(candidate_subscores))
+        score = float(raw_candidate.get("avg_reward", 0.0) or 0.0)
+        scores.append(score)
+        eval_counts.append(len(candidate_subscores))
+        metric_calls_total += len(candidate_subscores)
+
+    per_instance_best: dict[int, set[int]] = {}
+    for example_idx in range(len(eval_set)):
+        best_score = max((item.get(example_idx, float("-inf")) for item in subscores), default=float("-inf"))
+        per_instance_best[example_idx] = {
+            candidate_idx
+            for candidate_idx, item in enumerate(subscores)
+            if item.get(example_idx, float("-inf")) == best_score
+        }
+
+    if SynthGEPAResult is not None:
+        return SynthGEPAResult(
+            candidates=candidates,
+            parents=parents,
+            val_aggregate_scores=scores,
+            val_subscores=subscores,
+            per_val_instance_best_candidates=per_instance_best,
+            discovery_eval_counts=eval_counts,
+            total_metric_calls=metric_calls_total,
+        )
+
+    return LocalGEPAResult(
+        candidates=candidates,
+        parents=parents,
+        val_aggregate_scores=scores,
+        val_subscores=subscores,
+        per_val_instance_best_candidates=per_instance_best,
+        discovery_eval_counts=eval_counts,
+        total_metric_calls=metric_calls_total,
+    )
+
+
+__all__ = [
+    "LocalGEPAAdapterProtocol",
+    "LocalGEPAResult",
+    "SynthOfflineLearningAdapter",
+    "optimize",
+]

prompt_opt/mipro.py

@@ -0,0 +1,114 @@
+"""Local MIPRO entrypoint backed by the mirrored offline SDK runtime."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from prompt_opt.sdk.optimization.internal.prompt_learning import PromptLearningJob
+
+
+_PROPOSER_BACKENDS = ["single_prompt", "rlm"]
+
+
+def proposer_backends() -> list[str]:
+    """Return the supported public proposer backend tokens."""
+    return list(_PROPOSER_BACKENDS)
+
+
+def _policy_to_initial_candidate(initial_policy: dict[str, Any]) -> dict[str, Any]:
+    template = str(initial_policy.get("template", "")).strip() or "You are a helpful assistant."
+    return {
+        "stages": [
+            {
+                "id": "default",
+                "name": "Default",
+                "messages": [
+                    {"role": "system", "pattern": template, "order": 0},
+                    {"role": "user", "pattern": "{input}", "order": 1},
+                ],
+                "wildcards": {},
+            }
+        ]
+    }
+
+
+def _extract_best_policy(result_payload: dict[str, Any]) -> dict[str, Any]:
+    best_candidate = result_payload.get("best_candidate")
+    if not isinstance(best_candidate, dict):
+        return {"template": ""}
+    stages = best_candidate.get("stages") or best_candidate.get("candidate", {}).get("stages") or []
+    for stage in stages:
+        if not isinstance(stage, dict):
+            continue
+        for message in stage.get("messages", []):
+            if not isinstance(message, dict):
+                continue
+            if message.get("role") == "system":
+                text = message.get("pattern") or message.get("content")
+                if isinstance(text, str):
+                    return {"template": text}
+    content = best_candidate.get("candidate_content")
+    if isinstance(content, str):
+        return {"template": content}
+    return {"template": ""}
+
+
+def run_mipro(
+    *,
+    config: dict[str, Any],
+    initial_policy: dict[str, Any],
+    dataset: dict[str, Any],
+    task_llm: Callable[[str], str],
+) -> dict[str, Any]:
+    """Run local offline MIPRO and return a JSON-compatible result payload."""
+    examples = []
+    for item in dataset.get("examples", []):
+        if isinstance(item, dict):
+            examples.append(
+                {
+                    "input": item.get("input"),
+                    "answer": item.get("expected"),
+                    "expected": item.get("expected"),
+                    "metadata": dict(item.get("metadata") or {}),
+                }
+            )
+    prompt_learning_config = {
+        "prompt_learning": {
+            "algorithm": "mipro",
+            "execution_mode": "retrieved",
+            "task_data": {
+                "examples": examples,
+            },
+            "mipro": {
+                "initial_candidate": _policy_to_initial_candidate(initial_policy),
+                "num_candidates": int(config.get("num_candidates", 8)),
+                "max_iterations": int(config.get("max_iterations", 8)),
+                "early_stop_rounds": int(config.get("early_stop_rounds", 3)),
+                "min_improvement": float(config.get("min_improvement", 1e-6)),
+                "seed": int(config.get("seed", 0)),
+                "proposer_backend": str(config.get("proposer_backend", "single_prompt")),
+                "termination_conditions": {
+                    "total_rollouts": int(config.get("max_iterations", 8)) * max(1, len(examples)),
+                },
+            },
+            "local_runtime": {
+                "task_model": task_llm,
+            },
+        }
+    }
+
+    job = PromptLearningJob.from_dict(
+        prompt_learning_config,
+        backend_url="local://prompt-opt",
+        api_key="local",
+    )
+    job_id = job.submit()
+    result = job.stream_until_complete(timeout=300.0, interval=0.05).to_dict()
+    return {
+        "run_id": job_id,
+        "initial_policy": dict(initial_policy),
+        "best_policy": _extract_best_policy(result),
+        "best_score": result.get("best_reward"),
+        "history": job.get_results().get("result", {}).get("history", []),
+        "job_result": result,
+    }

prompt_opt/sdk/__init__.py

@@ -0,0 +1,2 @@
+"""Mirrored Synth-style SDK namespace for local prompt optimization."""
+

prompt_opt/sdk/optimization/__init__.py

@@ -0,0 +1,14 @@
+"""Local offline prompt optimization SDK."""
+
+from .models import PolicyCandidate, PolicyCandidatePage, PromptLearningResult
+from .policy.v1 import PolicyOptimizationOfflineJob
+from .internal.prompt_learning import PromptLearningJob, PromptLearningJobConfig
+
+__all__ = [
+    "PolicyCandidate",
+    "PolicyCandidatePage",
+    "PolicyOptimizationOfflineJob",
+    "PromptLearningJob",
+    "PromptLearningJobConfig",
+    "PromptLearningResult",
+]

prompt_opt/sdk/optimization/internal/__init__.py

@@ -0,0 +1,2 @@
+"""Internal mirrored SDK helpers."""
+

prompt_opt/sdk/optimization/internal/configs/__init__.py

@@ -0,0 +1,16 @@
+"""Mirrored prompt-learning config models."""
+
+from .prompt_learning import (
+    GEPAConfig,
+    MIPROAlgorithmConfig,
+    PromptLearningConfig,
+    PromptLearningPolicyConfig,
+)
+
+__all__ = [
+    "GEPAConfig",
+    "MIPROAlgorithmConfig",
+    "PromptLearningConfig",
+    "PromptLearningPolicyConfig",
+]
+

prompt_opt/sdk/optimization/internal/configs/prompt_learning.py

@@ -0,0 +1,230 @@
+"""Canonical prompt-learning config models for local offline prompt-opt."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from pydantic import Field, field_validator, model_validator
+
+from ..utils import load_toml
+from .shared import ExtraModel
+
+
+class PromptLearningPolicyConfig(ExtraModel):
+    provider: str | None = None
+    inference_url: str | None = None
+    inference_mode: str | None = None
+    temperature: float = 0.0
+    max_completion_tokens: int = 512
+    policy_name: str | None = None
+    config: dict[str, Any] = Field(default_factory=dict)
+    context_override: dict[str, Any] | None = None
+
+
+class MessagePatternConfig(ExtraModel):
+    role: str
+    pattern: str
+    order: int = 0
+
+
+class PromptStageConfig(ExtraModel):
+    id: str | None = None
+    name: str | None = None
+    messages: list[MessagePatternConfig] = Field(default_factory=list)
+    wildcards: dict[str, str] = Field(default_factory=dict)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class PromptCandidateConfig(ExtraModel):
+    stages: list[PromptStageConfig] = Field(default_factory=list)
+
+
+class TrainPoolsConfig(ExtraModel):
+    reflection_seeds: list[int] = Field(default_factory=list)
+    pareto_seeds: list[int] = Field(default_factory=list)
+    seeds: list[int] = Field(default_factory=list)
+
+
+class TaskDataConfig(ExtraModel):
+    split: str | None = None
+    train_pools: TrainPoolsConfig | dict[str, Any] | None = None
+    validation_seeds: list[int] = Field(default_factory=list)
+    validation_examples: list[dict[str, Any]] = Field(default_factory=list)
+    train_examples: list[dict[str, Any]] = Field(default_factory=list)
+    examples: list[dict[str, Any]] = Field(default_factory=list)
+    examples_by_seed: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+    @field_validator("train_pools", mode="before")
+    @classmethod
+    def _coerce_train_pools(cls, value: Any) -> Any:
+        if value is None or isinstance(value, TrainPoolsConfig):
+            return value
+        if isinstance(value, Mapping):
+            return TrainPoolsConfig.model_validate(dict(value))
+        raise ValueError("train_pools must be a mapping when provided")
+
+
+class PopulationConfig(ExtraModel):
+    initial_size: int = 1
+    num_generations: int = 1
+    children_per_generation: int = 4
+
+
+class TerminationConditionsConfig(ExtraModel):
+    total_rollouts: int | None = None
+    max_iterations: int | None = None
+    patience: int | None = None
+
+
+class GEPAConfig(ExtraModel):
+    env_name: str | None = None
+    initial_candidate: PromptCandidateConfig | dict[str, Any] | None = None
+    termination_conditions: TerminationConditionsConfig | dict[str, Any] | None = None
+    mode: str | None = None
+    execution_mode: str | None = None
+    proposer_backend: str | None = None
+    population: PopulationConfig | dict[str, Any] | None = None
+    modules: list[dict[str, Any]] | None = None
+
+    @field_validator("initial_candidate", mode="before")
+    @classmethod
+    def _coerce_initial_candidate(cls, value: Any) -> Any:
+        if value is None or isinstance(value, PromptCandidateConfig):
+            return value
+        if isinstance(value, Mapping):
+            return PromptCandidateConfig.model_validate(dict(value))
+        raise ValueError("initial_candidate must be a mapping when provided")
+
+    @field_validator("termination_conditions", mode="before")
+    @classmethod
+    def _coerce_termination(cls, value: Any) -> Any:
+        if value is None or isinstance(value, TerminationConditionsConfig):
+            return value
+        if isinstance(value, Mapping):
+            return TerminationConditionsConfig.model_validate(dict(value))
+        raise ValueError("termination_conditions must be a mapping when provided")
+
+    @field_validator("population", mode="before")
+    @classmethod
+    def _coerce_population(cls, value: Any) -> Any:
+        if value is None or isinstance(value, PopulationConfig):
+            return value
+        if isinstance(value, Mapping):
+            return PopulationConfig.model_validate(dict(value))
+        raise ValueError("population must be a mapping when provided")
+
+
+class MIPROAlgorithmConfig(ExtraModel):
+    initial_candidate: PromptCandidateConfig | dict[str, Any] | None = None
+    execution_mode: str | None = None
+    mode: str | None = None
+    num_candidates: int = 8
+    max_iterations: int = 8
+    early_stop_rounds: int = 3
+    min_improvement: float = 1e-6
+    proposer_backend: str = "single_prompt"
+    seed: int = 0
+    termination_conditions: TerminationConditionsConfig | dict[str, Any] | None = None
+
+    @field_validator("initial_candidate", mode="before")
+    @classmethod
+    def _coerce_initial_candidate(cls, value: Any) -> Any:
+        if value is None or isinstance(value, PromptCandidateConfig):
+            return value
+        if isinstance(value, Mapping):
+            return PromptCandidateConfig.model_validate(dict(value))
+        raise ValueError("initial_candidate must be a mapping when provided")
+
+    @field_validator("termination_conditions", mode="before")
+    @classmethod
+    def _coerce_termination(cls, value: Any) -> Any:
+        if value is None or isinstance(value, TerminationConditionsConfig):
+            return value
+        if isinstance(value, Mapping):
+            return TerminationConditionsConfig.model_validate(dict(value))
+        raise ValueError("termination_conditions must be a mapping when provided")
+
+
+class PromptLearningConfig(ExtraModel):
+    algorithm: str
+    job_kind: str | None = None
+    algorithm_name: str | None = None
+    execution_mode: str | None = None
+    config_schema_version: str | None = None
+    container_url: str | None = None
+    container_id: str | None = None
+    task_data: TaskDataConfig | dict[str, Any] | None = None
+    policy: PromptLearningPolicyConfig | dict[str, Any] | None = None
+    gepa: GEPAConfig | None = None
+    mipro: MIPROAlgorithmConfig | dict[str, Any] | None = None
+    verifier: dict[str, Any] | None = None
+    proxy_models: dict[str, Any] | None = None
+    env_config: dict[str, Any] | None = None
+    use_byok: bool | None = None
+
+    @field_validator("task_data", mode="before")
+    @classmethod
+    def _coerce_task_data(cls, value: Any) -> Any:
+        if value is None or isinstance(value, TaskDataConfig):
+            return value
+        if isinstance(value, Mapping):
+            return TaskDataConfig.model_validate(dict(value))
+        raise ValueError("task_data must be a mapping when provided")
+
+    @field_validator("policy", mode="before")
+    @classmethod
+    def _coerce_policy(cls, value: Any) -> Any:
+        if value is None or isinstance(value, PromptLearningPolicyConfig):
+            return value
+        if isinstance(value, Mapping):
+            return PromptLearningPolicyConfig.model_validate(dict(value))
+        raise ValueError("policy must be a mapping when provided")
+
+    @field_validator("mipro", mode="before")
+    @classmethod
+    def _coerce_mipro(cls, value: Any) -> Any:
+        if value is None or isinstance(value, MIPROAlgorithmConfig):
+            return value
+        if isinstance(value, Mapping):
+            return MIPROAlgorithmConfig.model_validate(dict(value))
+        raise ValueError("mipro must be a mapping when provided")
+
+    @model_validator(mode="after")
+    def _normalize_execution_mode(self) -> "PromptLearningConfig":
+        if self.algorithm_name is None:
+            self.algorithm_name = self.algorithm
+        if self.job_kind is None:
+            self.job_kind = "optimization"
+
+        algorithm_mode = None
+        if self.algorithm == "gepa" and self.gepa is not None:
+            algorithm_mode = self.gepa.execution_mode or self.gepa.mode
+        elif self.algorithm == "mipro" and self.mipro is not None:
+            algorithm_mode = self.mipro.execution_mode or self.mipro.mode
+
+        effective = self.execution_mode or algorithm_mode or "offline"
+        if effective == "proxied":
+            effective = "retrieved"
+        self.execution_mode = effective
+
+        if self.algorithm == "gepa" and self.gepa is not None:
+            self.gepa.execution_mode = effective
+        if self.algorithm == "mipro" and self.mipro is not None:
+            self.mipro.execution_mode = effective
+        return self
+
+    @classmethod
+    def from_mapping(cls, data: Mapping[str, Any]) -> "PromptLearningConfig":
+        payload = dict(data)
+        if "prompt_learning" in payload and isinstance(payload["prompt_learning"], Mapping):
+            payload = dict(payload["prompt_learning"])
+        return cls.model_validate(payload)
+
+    @classmethod
+    def from_path(cls, path: str | Path) -> "PromptLearningConfig":
+        return cls.from_mapping(load_toml(path))
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.model_dump(mode="python", exclude_none=True)

prompt_opt/sdk/optimization/internal/configs/shared.py

@@ -0,0 +1,12 @@
+"""Shared config base models."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ExtraModel(BaseModel):
+    """Base model that preserves unknown keys for forward compatibility."""
+
+    model_config = ConfigDict(extra="allow")
+