synth-optimizers 0.1.0__py3-none-any.whl

gepa/__init__.py

@@ -0,0 +1,5 @@
+"""Drop-in `gepa` module shim provided by prompt-opt."""
+
+from prompt_opt.gepa_ai_compat import LocalGEPAAdapterProtocol, optimize
+
+__all__ = ["LocalGEPAAdapterProtocol", "optimize"]

prompt_opt/__init__.py

@@ -0,0 +1,16 @@
+"""synth-optimizers public API exposed through the `prompt_opt` package."""
+
+from .gepa_ai_compat import LocalGEPAAdapterProtocol, optimize
+from .mipro import proposer_backends, run_mipro
+from .dspy.miprov2 import MIPROv2
+from .sdk.optimization import PolicyOptimizationOfflineJob, PromptLearningJob
+
+__all__ = [
+    "LocalGEPAAdapterProtocol",
+    "MIPROv2",
+    "PolicyOptimizationOfflineJob",
+    "PromptLearningJob",
+    "optimize",
+    "proposer_backends",
+    "run_mipro",
+]

prompt_opt/adapters/__init__.py

@@ -0,0 +1,18 @@
+"""Adapter implementations for prompt-opt."""
+
+from .synth_container import (
+    ContainerEvaluationBatch,
+    SynthContainerLearningAdapter,
+    default_rollout_request_builder,
+    default_rollout_score_extractor,
+)
+from .synth_offline import LocalEvaluator, SynthOfflineLearningAdapter
+
+__all__ = [
+    "ContainerEvaluationBatch",
+    "LocalEvaluator",
+    "SynthContainerLearningAdapter",
+    "SynthOfflineLearningAdapter",
+    "default_rollout_request_builder",
+    "default_rollout_score_extractor",
+]

prompt_opt/adapters/synth_container.py

@@ -0,0 +1,170 @@
+"""Synth Container adapter for local prompt-opt evaluations."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+from urllib import request
+
+
+RolloutRequestBuilder = Callable[[Mapping[str, Any], dict[str, str], str], dict[str, Any]]
+RolloutScoreExtractor = Callable[[dict[str, Any]], float]
+
+
+def default_rollout_request_builder(
+    *,
+    example: Mapping[str, Any],
+    candidate: dict[str, str],
+    trace_correlation_id: str,
+    env_name: str = "prompt-opt-local",
+    seed: int | None = None,
+) -> dict[str, Any]:
+    """Build a minimal Synth rollout request using the canonical container contract."""
+    env_seed = seed
+    if env_seed is None:
+        raw_seed = example.get("seed")
+        if isinstance(raw_seed, int):
+            env_seed = raw_seed
+
+    return {
+        "trace_correlation_id": trace_correlation_id,
+        "env": {
+            "env_name": env_name,
+            "seed": env_seed,
+            "config": {
+                "example": dict(example),
+            },
+        },
+        "policy": {
+            "policy_name": "prompt-opt",
+            "config": {
+                "candidate": dict(candidate),
+            },
+        },
+        "on_done": "reset",
+        "safety": {"max_time_s": 300},
+    }
+
+
+def default_rollout_score_extractor(response_payload: dict[str, Any]) -> float:
+    """Extract the rollout reward from a Synth Container response payload."""
+    reward_info = response_payload.get("metrics")
+    if not isinstance(reward_info, dict):
+        reward_info = response_payload.get("reward_info")
+    if not isinstance(reward_info, dict):
+        raise ValueError("rollout response is missing metrics/reward_info")
+    reward = reward_info.get("outcome_reward")
+    if reward is None:
+        raise ValueError("rollout response metrics are missing outcome_reward")
+    return float(reward)
+
+
+@dataclass(frozen=True)
+class ContainerEvaluationBatch:
+    """Batch output shaped to GEPA adapter expectations."""
+
+    outputs: list[dict[str, Any]]
+    scores: list[float]
+    trajectories: list[dict[str, Any] | None] | None = None
+    objective_scores: list[dict[str, float]] | None = None
+
+
+class SynthContainerLearningAdapter:
+    """Evaluate candidates against a Synth-compatible Container rollout endpoint."""
+
+    def __init__(
+        self,
+        *,
+        container_url: str,
+        request_builder: RolloutRequestBuilder,
+        api_key: str | None = None,
+        headers: Mapping[str, str] | None = None,
+        timeout_seconds: float = 30.0,
+        score_extractor: RolloutScoreExtractor = default_rollout_score_extractor,
+    ) -> None:
+        self._container_url = container_url.rstrip("/")
+        self._request_builder = request_builder
+        self._api_key = api_key
+        self._headers = dict(headers or {})
+        self._timeout_seconds = float(timeout_seconds)
+        self._score_extractor = score_extractor
+
+    def _post_rollout(self, payload: dict[str, Any]) -> dict[str, Any]:
+        body = json.dumps(payload).encode("utf-8")
+        headers = {"content-type": "application/json", **self._headers}
+        if self._api_key:
+            headers["x-api-key"] = self._api_key
+        http_request = request.Request(
+            f"{self._container_url}/rollout",
+            data=body,
+            headers=headers,
+            method="POST",
+        )
+        with request.urlopen(http_request, timeout=self._timeout_seconds) as response:
+            return json.loads(response.read().decode("utf-8"))
+
+    def evaluate(
+        self,
+        batch: list[Mapping[str, Any]],
+        candidate: dict[str, str],
+        capture_traces: bool = False,
+    ) -> ContainerEvaluationBatch:
+        outputs: list[dict[str, Any]] = []
+        scores: list[float] = []
+        trajectories: list[dict[str, Any] | None] = []
+        objective_scores: list[dict[str, float]] = []
+
+        for index, example in enumerate(batch):
+            correlation_id = f"prompt-opt-{index}-{uuid.uuid4().hex}"
+            rollout_request = self._request_builder(example, candidate, correlation_id)
+            rollout_response = self._post_rollout(rollout_request)
+            score = self._score_extractor(rollout_response)
+            reward_info = rollout_response.get("metrics") or rollout_response.get("reward_info") or {}
+            outcome_objectives = reward_info.get("outcome_objectives")
+            outputs.append(
+                {
+                    "request": rollout_request,
+                    "response": rollout_response,
+                }
+            )
+            scores.append(score)
+            trajectories.append(rollout_response.get("trace") if capture_traces else None)
+            objective_scores.append(
+                dict(outcome_objectives) if isinstance(outcome_objectives, dict) else {"reward": score}
+            )
+
+        return ContainerEvaluationBatch(
+            outputs=outputs,
+            scores=scores,
+            trajectories=trajectories,
+            objective_scores=objective_scores,
+        )
+
+    def make_reflective_dataset(
+        self,
+        candidate: dict[str, str],
+        eval_batch: ContainerEvaluationBatch,
+        components_to_update: list[str],
+    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
+        reflective_rows: list[dict[str, Any]] = []
+        for idx, (output, score) in enumerate(zip(eval_batch.outputs, eval_batch.scores)):
+            reflective_rows.append(
+                {
+                    "index": idx,
+                    "candidate": dict(candidate),
+                    "score": float(score),
+                    "output": output,
+                }
+            )
+        return {component: tuple(reflective_rows) for component in components_to_update}
+
+
+__all__ = [
+    "ContainerEvaluationBatch",
+    "SynthContainerLearningAdapter",
+    "default_rollout_request_builder",
+    "default_rollout_score_extractor",
+]

prompt_opt/adapters/synth_offline.py

@@ -0,0 +1,81 @@
+"""Synth-compatible offline learning adapters for local use."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Callable
+
+
+ScoreFunction = Callable[[Mapping[str, Any], dict[str, str]], float]
+
+
+@dataclass(frozen=True)
+class LocalEvaluator:
+    """Evaluates one dataset example against one candidate."""
+
+    score_fn: ScoreFunction
+
+    def score_batch(self, batch: Sequence[Mapping[str, Any]], candidate: dict[str, str]) -> list[float]:
+        """Return per-example scores for the candidate."""
+        return [float(self.score_fn(example, candidate)) for example in batch]
+
+
+@dataclass(frozen=True)
+class LocalEvaluationBatch:
+    """Local batch output shaped to GEPA adapter expectations."""
+
+    outputs: list[dict[str, Any]]
+    scores: list[float]
+    trajectories: list[dict[str, Any]] | None = None
+    objective_scores: list[dict[str, float]] | None = None
+
+
+class SynthOfflineLearningAdapter:
+    """Local adapter with a GEPA-compatible evaluation shape.
+
+    This adapter is intentionally lightweight so it can run completely offline
+    without backend dependencies.
+    """
+
+    def __init__(self, evaluator: LocalEvaluator) -> None:
+        self._evaluator = evaluator
+
+    def evaluate(
+        self,
+        batch: list[Mapping[str, Any]],
+        candidate: dict[str, str],
+        capture_traces: bool = False,
+    ) -> LocalEvaluationBatch:
+        del capture_traces
+        scores = self._evaluator.score_batch(batch, candidate)
+        outputs = [{"candidate": candidate, "score": score} for score in scores]
+        trajectories = [{"kind": "offline_local"} for _ in scores]
+        objective_scores = [{"reward": score} for score in scores]
+        return LocalEvaluationBatch(
+            outputs=outputs,
+            scores=scores,
+            trajectories=trajectories,
+            objective_scores=objective_scores,
+        )
+
+    def make_reflective_dataset(
+        self,
+        candidate: dict[str, str],
+        eval_batch: LocalEvaluationBatch,
+        components_to_update: list[str],
+    ) -> Mapping[str, Sequence[Mapping[str, Any]]]:
+        reflective_rows: list[dict[str, Any]] = []
+        for idx, (output, score) in enumerate(zip(eval_batch.outputs, eval_batch.scores)):
+            reflective_rows.append(
+                {
+                    "index": idx,
+                    "candidate": dict(candidate),
+                    "score": float(score),
+                    "output": output,
+                }
+            )
+        return {
+            component: tuple(reflective_rows)
+            for component in components_to_update
+        }

prompt_opt/dspy/__init__.py

@@ -0,0 +1,5 @@
+"""DSPy-facing compatibility surface for prompt-opt."""
+
+from .miprov2 import MIPROv2
+
+__all__ = ["MIPROv2"]

prompt_opt/dspy/gepa.py

@@ -0,0 +1,5 @@
+"""DSPy + GEPA slot-in helpers."""
+
+from prompt_opt.gepa_ai_compat import optimize
+
+__all__ = ["optimize"]

prompt_opt/dspy/miprov2.py

@@ -0,0 +1,281 @@
+"""DSPy-compatible MIPROv2 wrapper backed by the local offline SDK."""
+
+from __future__ import annotations
+
+import copy
+import json
+import math
+from typing import Any, Literal
+
+from prompt_opt.mipro import proposer_backends
+from prompt_opt.sdk.optimization.internal.prompt_learning import PromptLearningJob
+
+
+def _extract_text_field(example: Any, keys: tuple[str, ...]) -> str:
+    if isinstance(example, dict):
+        for key in keys:
+            value = example.get(key)
+            if value is not None:
+                return str(value)
+    for key in keys:
+        if hasattr(example, key):
+            value = getattr(example, key)
+            if value is not None:
+                return str(value)
+    return ""
+
+
+def _materialize_dataset(dataset: list[Any]) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for item in dataset:
+        input_text = _extract_text_field(item, ("input", "question", "query", "text"))
+        answer_text = _extract_text_field(item, ("answer", "label", "expected", "output"))
+        additional_context: dict[str, Any] = {}
+        if isinstance(item, dict):
+            raw_context = item.get("additional_context")
+            if isinstance(raw_context, dict):
+                additional_context = dict(raw_context)
+        labels_blob = str(additional_context.get("labels", "")).strip()
+        if labels_blob:
+            input_text = f"{input_text}\n\nLabels: {labels_blob}"
+        rows.append({"input": input_text, "answer": answer_text, "metadata": {}})
+    return rows
+
+
+def _resolve_num_trials(
+    *,
+    auto: Literal["light", "medium", "heavy"] | None,
+    explicit_trials: int | None,
+    num_candidates: int | None,
+    student: Any,
+) -> int:
+    if explicit_trials is not None:
+        return max(1, int(explicit_trials))
+    if auto is None and num_candidates is not None:
+        predictor_count = 1
+        named_predictors = getattr(student, "named_predictors", None)
+        if callable(named_predictors):
+            try:
+                predictor_count = max(1, len(list(named_predictors())))
+            except Exception:
+                predictor_count = 1
+        suggested = int(max(2 * (predictor_count * 2) * math.log2(max(num_candidates, 2)), 1.5 * num_candidates))
+        return max(1, suggested)
+    if auto == "light":
+        return 7
+    if auto == "medium":
+        return 20
+    if auto == "heavy":
+        return 50
+    raise ValueError("num_trials must be provided when auto is None.")
+
+
+def _extract_seed_candidate_from_student(student: Any) -> dict[str, str]:
+    named_predictors = getattr(student, "named_predictors", None)
+    if callable(named_predictors):
+        try:
+            entries = list(named_predictors())
+            candidate: dict[str, str] = {}
+            for entry_index, (name, predictor) in enumerate(entries):
+                signature = getattr(predictor, "signature", None)
+                instructions = getattr(signature, "instructions", None)
+                if isinstance(instructions, str) and instructions.strip():
+                    candidate[str(name or f"predictor_{entry_index}")] = instructions.strip()
+            if candidate:
+                return candidate
+        except Exception:
+            pass
+    return {"default": "You are a helpful assistant."}
+
+
+def _apply_prompts_to_student(student: Any, prompts: dict[str, str]) -> Any:
+    named_predictors = getattr(student, "named_predictors", None)
+    if not callable(named_predictors):
+        return student
+    try:
+        for name, predictor in list(named_predictors()):
+            prompt = prompts.get(str(name))
+            if not isinstance(prompt, str) or not prompt.strip():
+                continue
+            signature = getattr(predictor, "signature", None)
+            if signature is None:
+                continue
+            with_instructions = getattr(signature, "with_instructions", None)
+            if callable(with_instructions):
+                predictor.signature = with_instructions(prompt)
+            else:
+                signature.instructions = prompt
+    except Exception:
+        return student
+    return student
+
+
+def _extract_best_prompts(result_payload: dict[str, Any]) -> dict[str, str]:
+    best_candidate = result_payload.get("best_candidate")
+    if not isinstance(best_candidate, dict):
+        return {}
+    stage_items = best_candidate.get("stages") or best_candidate.get("candidate", {}).get("stages") or []
+    prompts: dict[str, str] = {}
+    for stage in stage_items:
+        if not isinstance(stage, dict):
+            continue
+        stage_key = str(stage.get("id") or stage.get("name") or f"stage_{len(prompts)}")
+        for message in stage.get("messages", []):
+            if isinstance(message, dict) and message.get("role") == "system":
+                text = message.get("pattern") or message.get("content")
+                if isinstance(text, str) and text.strip():
+                    prompts[stage_key] = text.strip()
+                    break
+    return prompts
+
+
+class MIPROv2:
+    """Drop-in replacement for `dspy.MIPROv2` running locally only."""
+
+    def __init__(
+        self,
+        metric: Any,
+        prompt_model: Any | None = None,
+        task_model: Any | None = None,
+        teacher_settings: dict[str, Any] | None = None,
+        max_bootstrapped_demos: int = 4,
+        max_labeled_demos: int = 4,
+        auto: Literal["light", "medium", "heavy"] | None = "light",
+        num_candidates: int | None = None,
+        num_threads: int | None = None,
+        max_errors: int | None = None,
+        seed: int = 9,
+        init_temperature: float = 1.0,
+        verbose: bool = False,
+        track_stats: bool = True,
+        log_dir: str | None = None,
+        metric_threshold: float | None = None,
+        backend_mode: Literal["local"] = "local",
+        proposer_backend: Literal["single_prompt", "rlm"] = "single_prompt",
+        **kwargs: Any,
+    ) -> None:
+        del metric, teacher_settings, num_threads, max_errors, init_temperature, log_dir, metric_threshold, kwargs
+        if proposer_backend not in proposer_backends():
+            raise ValueError(f"Unsupported proposer_backend={proposer_backend!r}")
+        self.prompt_model = prompt_model
+        self.task_model = task_model
+        self.max_bootstrapped_demos = max_bootstrapped_demos
+        self.max_labeled_demos = max_labeled_demos
+        self.auto = auto
+        self.num_candidates = num_candidates
+        self.seed = seed
+        self.verbose = verbose
+        self.track_stats = track_stats
+        self.backend_mode = backend_mode
+        self.proposer_backend = proposer_backend
+
+    def compile(
+        self,
+        student: Any,
+        *,
+        trainset: list[Any],
+        teacher: Any = None,
+        valset: list[Any] | None = None,
+        num_trials: int | None = None,
+        num_candidates: int | None = None,
+        max_bootstrapped_demos: int | None = None,
+        max_labeled_demos: int | None = None,
+        seed: int | None = None,
+        minibatch: bool = True,
+        minibatch_size: int = 35,
+        minibatch_full_eval_steps: int = 5,
+        program_aware_proposer: bool = True,
+        data_aware_proposer: bool = True,
+        view_data_batch_size: int = 10,
+        tip_aware_proposer: bool = True,
+        fewshot_aware_proposer: bool = True,
+        requires_permission_to_run: bool | None = None,
+        provide_traceback: bool | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        del (
+            teacher,
+            max_bootstrapped_demos,
+            max_labeled_demos,
+            minibatch,
+            minibatch_size,
+            minibatch_full_eval_steps,
+            program_aware_proposer,
+            data_aware_proposer,
+            view_data_batch_size,
+            tip_aware_proposer,
+            fewshot_aware_proposer,
+            requires_permission_to_run,
+            provide_traceback,
+        )
+        if self.backend_mode != "local":
+            raise ValueError("prompt-opt is local-only. Use backend_mode='local'.")
+
+        if not callable(getattr(self.task_model, "__call__", None)):
+            raise ValueError(
+                "Local backend_mode requires task_model to be callable(prompt)->str. "
+                "Model-id strings are not supported in local-only mode."
+            )
+
+        run_seed = int(seed if seed is not None else self.seed)
+        train_records = _materialize_dataset(trainset)
+        val_records = _materialize_dataset(valset) if valset is not None else train_records
+        effective_num_candidates = int(num_candidates) if num_candidates is not None else self.num_candidates
+        trials = _resolve_num_trials(
+            auto=self.auto,
+            explicit_trials=num_trials,
+            num_candidates=effective_num_candidates,
+            student=student,
+        )
+        seed_candidate = _extract_seed_candidate_from_student(student)
+        stages = [
+            {
+                "id": stage_id,
+                "name": stage_id,
+                "messages": [
+                    {"role": "system", "pattern": prompt, "order": 0},
+                    {"role": "user", "pattern": "{input}", "order": 1},
+                ],
+                "wildcards": {},
+            }
+            for stage_id, prompt in seed_candidate.items()
+        ]
+        prompt_learning_config = {
+            "prompt_learning": {
+                "algorithm": "mipro",
+                "execution_mode": "retrieved",
+                "task_data": {
+                    "train_examples": train_records,
+                    "validation_examples": val_records,
+                },
+                "mipro": {
+                    "initial_candidate": {"stages": stages},
+                    "num_candidates": max(1, int(effective_num_candidates or 8)),
+                    "max_iterations": max(1, int(trials)),
+                    "early_stop_rounds": 3,
+                    "min_improvement": 1e-6,
+                    "seed": run_seed,
+                    "proposer_backend": self.proposer_backend,
+                    "termination_conditions": {
+                        "total_rollouts": max(1, int(trials)) * max(1, len(val_records)),
+                    },
+                },
+                "local_runtime": {
+                    "task_model": self.task_model,
+                },
+            }
+        }
+        job = PromptLearningJob.from_dict(
+            prompt_learning_config,
+            backend_url="local://prompt-opt",
+            api_key="local",
+        )
+        job.submit()
+        result = job.stream_until_complete(timeout=300.0, interval=0.05).to_dict()
+        optimized_program = copy.deepcopy(student)
+        best_prompts = _extract_best_prompts(result)
+        if best_prompts:
+            optimized_program = _apply_prompts_to_student(optimized_program, best_prompts)
+        if self.track_stats:
+            setattr(optimized_program, "prompt_opt_result", json.loads(json.dumps(result)))
+        return optimized_program