synth-ai 0.2.4.dev4__py3-none-any.whl → 0.2.4.dev6__py3-none-any.whl

synth_ai/learning/prompts/mipro.py

@@ -1,3 +1,275 @@
+"""
+MIPROv2-style prompt optimizer (modular, DSPy-inspired).
+
+This module provides a modular implementation of the MIPROv2 pseudocode from DSPy,
+adapted to a provider-agnostic "program" interface. The goal is to keep the
+bootstrapping and search process pluggable so it can be swapped for alternatives.
+
+Key ideas
+- Program adapter: unify how we set instructions/demos and run predictions.
+- Demo bootstrapping: gather high-confidence examples (by metric) as candidates.
+- Instruction proposals: generated by a prompt model from contextual summaries.
+- Search (placeholder): random/Bayesian-like search over (instructions × demos).
+
+Notes
+- The implementation is intentionally lightweight and dependency-free.
+- "BayesOpt" here is a placeholder randomized proposer that uses history; you
+  can plug in a real optimizer later.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, replace
+from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Sequence, Tuple
+
+
+# ---------------------------
+# Program adapter and protocols
+# ---------------------------
+
+
+class PredictProgram(Protocol):
+    """Minimal protocol a program must satisfy for MIPRO.
+
+    You can adapt your own pipeline to this by implementing these methods or
+    by wrapping it with `ProgramAdapter` below.
+    """
+
+    def deepcopy(self) -> "PredictProgram": ...
+
+    def run(self, x: Any, *, model: Optional[Any] = None) -> Any: ...
+
+    def with_instructions(self, instructions: Dict[str, str]) -> "PredictProgram": ...
+
+    def with_demos(self, demos: List[Tuple[Any, Any]]) -> "PredictProgram": ...
+
+    @property
+    def predictors(self) -> List[str]: ...
+
+
+@dataclass
+class ProgramAdapter:
+    """Adapter that turns a set of callables/state into a `PredictProgram`.
+
+    - run_fn: Callable[[x, model], y]
+    - state: arbitrary dict; supports `instructions` and `demos` keys
+    - predictors: list of predictor identifiers (e.g., names of prompt blocks)
+    - set_instructions: Callable to update instructions (per predictor)
+    - set_demos: Callable to update demos (global or per predictor)
+    """
+
+    run_fn: Callable[[Any, Optional[Any]], Any]
+    state: Dict[str, Any]
+    _predictors: List[str]
+    set_instructions: Callable[[Dict[str, str], Dict[str, Any]], Dict[str, Any]]
+    set_demos: Callable[[List[Tuple[Any, Any]], Dict[str, Any]], Dict[str, Any]]
+
+    def deepcopy(self) -> "ProgramAdapter":
+        return replace(self, state={**self.state})
+
+    def run(self, x: Any, *, model: Optional[Any] = None) -> Any:
+        return self.run_fn(x, model)
+
+    def with_instructions(self, instructions: Dict[str, str]) -> "ProgramAdapter":
+        new_state = self.set_instructions(instructions, {**self.state})
+        return replace(self, state=new_state)
+
+    def with_demos(self, demos: List[Tuple[Any, Any]]) -> "ProgramAdapter":
+        new_state = self.set_demos(demos, {**self.state})
+        return replace(self, state=new_state)
+
+    @property
+    def predictors(self) -> List[str]:
+        return list(self._predictors)
+
+
+# ---------------------------
+# Utility helpers
+# ---------------------------
+
+
+def summarize_dataset(trainset: Sequence[Tuple[Any, Any]], max_items: int = 50) -> str:
+    n = len(trainset)
+    ex = ", ".join(repr(trainset[i][0])[:40] for i in range(0, min(max_items, n), max(1, n // max_items or 1)))
+    return f"Dataset size: {n}. Example inputs: {ex}"
+
+
+def summarize_program(prog: PredictProgram) -> str:
+    return f"Program predictors: {prog.predictors}"
+
+
+def random_tip(rng: random.Random) -> str:
+    tips = [
+        "Be concise.",
+        "Focus on the task definition.",
+        "Use the provided examples as guidance.",
+        "Avoid unnecessary verbosity.",
+    ]
+    return rng.choice(tips)
+
+
+def choose(items: Sequence[Any], rng: Optional[random.Random] = None) -> Any:
+    r = rng or random
+    return r.choice(items)
+
+
+# ---------------------------
+# Evaluator
+# ---------------------------
+
+
+@dataclass
+class EvalResult:
+    score: float
+    subscores: List[float]
+
+
+def evaluate_program(program: PredictProgram, dataset: Sequence[Tuple[Any, Any]], metric: Callable[[Any, Any], float]) -> EvalResult:
+    subs = []
+    for x, y in dataset:
+        yhat = program.run(x)
+        subs.append(metric(yhat, y))
+    return EvalResult(score=float(sum(subs)) / max(1, len(subs)), subscores=subs)
+
+
+# ---------------------------
+# MIPROv2 compile
+# ---------------------------
+
+
+def mipro_v2_compile(
+    student: PredictProgram,
+    trainset: Sequence[Tuple[Any, Any]],
+    valset: Sequence[Tuple[Any, Any]],
+    metric: Callable[[Any, Any], float],
+    *,
+    prompt_model: Any,
+    task_model: Any,
+    max_bootstrapped_demos: int = 8,
+    max_labeled_demos: int = 4,
+    num_candidates: int = 8,
+    num_trials: int = 20,
+    minibatch: bool = True,
+    minibatch_size: int = 16,
+    minibatch_full_eval_steps: int = 5,
+    seed: int = 0,
+    auto: str = "light",
+    program_aware: bool = True,
+    data_aware: bool = True,
+    tip_aware: bool = True,
+    fewshot_aware: bool = True,
+) -> Tuple[PredictProgram, List[Dict[str, Any]]]:
+    """MIPROv2-style optimizer.
+
+    Arguments mirror the DSPy pseudocode but remain provider-agnostic. The
+    `prompt_model` must expose `generate_instructions(ctx, k)`; the `student`
+    program must implement the `PredictProgram` protocol.
+    """
+
+    rng = random.Random(seed)
+    program = student.deepcopy()
+
+    # Step 1: bootstrap few-shot example candidates
+    demo_candidates: List[Dict[str, Any]] = []
+    for _ in range(num_candidates):
+        boot: List[Tuple[Any, Any]] = []
+        # collect bootstrapped, self-consistent demos
+        while len(boot) < max_bootstrapped_demos:
+            x, y = rng.choice(trainset)
+            yhat = program.run(x, model=task_model)
+            if metric(yhat, y) == 1:  # perfect match
+                boot.append((x, y))
+        labeled = rng.sample(list(trainset), k=min(max_labeled_demos, len(trainset)))
+        demo_candidates.append({"boot": boot, "labeled": labeled})
+
+    # Step 2: propose instruction candidates per predictor
+    instr_candidates: Dict[str, List[str]] = {}
+    for pred in (program.predictors or ["predictor"]):
+        ctx: Dict[str, Any] = {}
+        if data_aware:
+            ctx["dataset_summary"] = summarize_dataset(trainset)
+        if program_aware:
+            ctx["program_summary"] = summarize_program(program)
+        if fewshot_aware and demo_candidates:
+            ctx["examples"] = choose(demo_candidates, rng)
+        if tip_aware:
+            ctx["tip"] = random_tip(rng)
+        cand = prompt_model.generate_instructions(ctx, k=num_candidates)
+        instr_candidates[pred] = list(cand)
+
+    # Step 3: Bayesian-optimization-like search (random proposer placeholder)
+    history: List[Tuple[Dict[str, Any], float]] = []
+    records: List[Dict[str, Any]] = []
+    best_score = -1.0
+    best_cfg: Optional[Dict[str, Any]] = None
+
+    def propose(history_: List[Tuple[Dict[str, Any], float]]) -> Dict[str, Any]:
+        # Placeholder: randomly sample from the cartesian product
+        instructions = {pred: choose(instr_candidates[pred], rng) for pred in instr_candidates}
+        demos = choose(demo_candidates, rng) if demo_candidates else None
+        return {"instructions": instructions, "demo_set": demos}
+
+    for t in range(1, num_trials + 1):
+        theta = propose(history)
+        program_t = program.with_instructions(theta["instructions"])
+        if theta.get("demo_set") is not None:
+            # Combine bootstrapped + labeled demos
+            ds = theta["demo_set"]
+            demo_set = list(ds.get("boot", [])) + list(ds.get("labeled", []))
+            program_t = program_t.with_demos(demo_set)
+
+        batch = (
+            valset
+            if not minibatch
+            else random.sample(list(valset), k=min(minibatch_size, len(valset)))
+        )
+        batch_res = evaluate_program(program_t, batch, metric)
+        s_t = batch_res.score
+        history.append((theta, s_t))
+        records.append({
+            "trial": t,
+            "evaluation": "batch" if minibatch else "full",
+            "score": s_t,
+            "intervention": {
+                "instructions": theta.get("instructions"),
+                "demo_set": theta.get("demo_set"),
+            },
+        })
+
+        if (not minibatch) or (t % max(1, minibatch_full_eval_steps) == 0):
+            full_res = evaluate_program(program_t, valset, metric)
+            s_full = full_res.score
+            if s_full > best_score:
+                best_score = s_full
+                best_cfg = theta
+            records.append({
+                "trial": t,
+                "evaluation": "full",
+                "score": s_full,
+                "intervention": {
+                    "instructions": theta.get("instructions"),
+                    "demo_set": theta.get("demo_set"),
+                },
+            })
+
+    if best_cfg is None:
+        return program, records
+
+    best_program = program.with_instructions(best_cfg["instructions"])
+    if best_cfg.get("demo_set") is not None:
+        ds = best_cfg["demo_set"]
+        demo_set = list(ds.get("boot", [])) + list(ds.get("labeled", []))
+        best_program = best_program.with_demos(demo_set)
+    return best_program, records
+
+
+__all__ = [
+    "PredictProgram",
+    "ProgramAdapter",
+    "evaluate_program",
+    "mipro_v2_compile",
+]
 
 
 class ExampleTwoStepDag:
@@ -5,4 +277,4 @@ class ExampleTwoStepDag:
 
 """
 A -> B
-"""
+"""

synth_ai/learning/prompts/random_search.py

@@ -0,0 +1,247 @@
+"""
+Random-search prompt optimizer (BootstrapFewShotWithRandomSearch), DSPy-inspired.
+
+Implements the high-level pseudocode of DSPy's Random Search optimizer in a
+provider-agnostic, modular style. You can plug in your own student/program and
+metric, and this module will explore baselines and bootstrapped few-shot variants.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+
+
+# ---------------------------
+# Protocol-like expectations (duck-typed)
+# ---------------------------
+
+
+class _ProgramLike:
+    def reset_copy(self):  # zero-shot copy
+        return self
+
+    def deepcopy(self):  # deep copy
+        return self
+
+    def with_demos(self, demos: List[Tuple[Any, Any]]):
+        return self
+
+    def run(self, x: Any) -> Any:
+        raise NotImplementedError
+
+
+# ---------------------------
+# Helpers and lightweight components
+# ---------------------------
+
+
+@dataclass
+class EvalResult:
+    score: float
+    subscores: List[float]
+
+
+def evaluate(program: _ProgramLike, dataset: Sequence[Tuple[Any, Any]], metric: Callable[[Any, Any], float]) -> EvalResult:
+    subs = []
+    for x, y in dataset:
+        subs.append(metric(program.run(x), y))
+    return EvalResult(sum(subs) / max(1, len(subs)), subs)
+
+
+class LabeledFewShot:
+    def __init__(self, k: int):
+        self.k = k
+
+    def compile(self, student: _ProgramLike, trainset: Sequence[Tuple[Any, Any]], sample: bool = True) -> _ProgramLike:
+        p = getattr(student, "deepcopy", student.reset_copy)()
+        demos = list(trainset)
+        if sample:
+            random.shuffle(demos)
+        p = p.with_demos(demos[: min(self.k, len(demos))])
+        return p
+
+
+class BootstrapFewShot:
+    def __init__(
+        self,
+        *,
+        metric: Callable[[Any, Any], float],
+        metric_threshold: Optional[float] = None,
+        max_bootstrapped_demos: int = 8,
+        max_labeled_demos: int = 0,
+        teacher_settings: Optional[Dict[str, Any]] = None,
+        max_rounds: int = 1,
+    ):
+        self.metric = metric
+        self.metric_threshold = metric_threshold
+        self.max_bootstrapped_demos = max_bootstrapped_demos
+        self.max_labeled_demos = max_labeled_demos
+        self.teacher_settings = teacher_settings or {}
+        self.max_rounds = max_rounds
+
+    def compile(
+        self,
+        student: _ProgramLike,
+        teacher: Optional[_ProgramLike],
+        trainset: Sequence[Tuple[Any, Any]],
+    ) -> _ProgramLike:
+        p = getattr(student, "deepcopy", student.reset_copy)()
+        rng = random.Random()
+        # If bootstrapped demos disabled, return labeled-only few-shot quickly
+        if self.max_bootstrapped_demos <= 0:
+            demos: List[Tuple[Any, Any]] = []
+            if self.max_labeled_demos > 0:
+                demos += rng.sample(list(trainset), k=min(self.max_labeled_demos, len(trainset)))
+            return p.with_demos(demos)
+        boot: List[Tuple[Any, Any]] = []
+        # Bootstrap demos by self consistency
+        for _ in range(self.max_rounds):
+            rng.shuffle(trainset := list(trainset))
+            for x, y in trainset:
+                yhat = p.run(x)
+                ok = self.metric(yhat, y)
+                if (self.metric_threshold is None and ok == 1) or (
+                    self.metric_threshold is not None and ok >= self.metric_threshold
+                ):
+                    boot.append((x, y))
+                if len(boot) >= self.max_bootstrapped_demos:
+                    break
+            if len(boot) >= self.max_bootstrapped_demos:
+                break
+
+        # Optionally add labeled demos
+        demos = list(boot)
+        if self.max_labeled_demos > 0:
+            demos += rng.sample(list(trainset), k=min(self.max_labeled_demos, len(trainset)))
+
+        return p.with_demos(demos)
+
+
+# ---------------------------
+# Random-search compile (BootstrapFewShotWithRandomSearch)
+# ---------------------------
+
+
+@dataclass
+class Candidate:
+    score: float
+    subscores: List[float]
+    seed: int
+    program: _ProgramLike
+
+
+def random_search_compile(
+    student: _ProgramLike,
+    trainset: Sequence[Tuple[Any, Any]],
+    valset: Sequence[Tuple[Any, Any]],
+    metric: Callable[[Any, Any], float],
+    *,
+    max_bootstrapped_demos: int = 8,
+    max_labeled_demos: int = 4,
+    max_rounds: int = 2,
+    num_candidate_programs: int = 16,
+    stop_at_score: Optional[float] = None,
+    evaluate_fn: Optional[Callable[[
+        _ProgramLike,
+        Sequence[Tuple[Any, Any]],
+        Callable[[Any, Any], float]
+    ], EvalResult]] = None,
+    on_candidate_evaluated: Optional[Callable[[int, float, EvalResult, Dict[str, Any]], None]] = None,
+) -> Tuple[_ProgramLike, List[Dict[str, Any]]]:
+    best_program: Optional[_ProgramLike] = None
+    best_score = float("-inf")
+    candidates: List[Candidate] = []
+    records: List[Dict[str, Any]] = []
+
+    seeds = list(range(num_candidate_programs))
+    seeds = [-3, -2, -1] + seeds  # zero-shot, labeled few-shot, bootstrapped few-shot
+
+    rng = random.Random(0)
+    for idx, seed in enumerate(seeds):
+        train_copy = list(trainset)
+
+        if seed == -3:
+            program = getattr(student, "reset_copy", student.deepcopy)()
+
+        elif seed == -2:
+            program = LabeledFewShot(k=max_labeled_demos).compile(student, train_copy, sample=True)
+
+        else:
+            if seed >= 0:
+                rng.shuffle(train_copy)
+            if max_bootstrapped_demos <= 0:
+                size = 0
+            else:
+                size = max_bootstrapped_demos if seed == -1 else rng.randint(1, max_bootstrapped_demos)
+            program = BootstrapFewShot(
+                metric=metric,
+                metric_threshold=None,
+                max_bootstrapped_demos=size,
+                max_labeled_demos=max_labeled_demos,
+                teacher_settings={},
+                max_rounds=max_rounds,
+            ).compile(student, teacher=None, trainset=train_copy)
+
+        res = (evaluate_fn(program, valset, metric) if evaluate_fn else evaluate(program, valset, metric))
+        cand = Candidate(score=res.score, subscores=res.subscores, seed=seed, program=program)
+        candidates.append(cand)
+        # Record an intervention summary for reproducibility
+        intervention: Dict[str, Any] = {"seed": seed}
+        if hasattr(program, "demos"):
+            try:
+                intervention["demos"] = getattr(program, "demos")  # type: ignore
+            except Exception:
+                intervention["demos"] = None
+        # Type of candidate
+        if seed == -3:
+            intervention["kind"] = "zero_shot"
+            intervention["label"] = "zero-shot"
+        elif seed == -2:
+            intervention["kind"] = "labeled_few_shot"
+            intervention["label"] = f"labeled-{max_labeled_demos}"
+        else:
+            intervention["kind"] = "bootstrapped_few_shot"
+            bs = 0
+            try:
+                # try to infer from program demos length if present
+                bs = len(intervention.get("demos") or [])
+            except Exception:
+                bs = 0
+            intervention["label"] = f"boot-b{max_bootstrapped_demos}-l{max_labeled_demos}"
+        record_obj = {
+            "score": cand.score,
+            "subscores": cand.subscores,
+            "intervention": intervention,
+        }
+        records.append(record_obj)
+
+        if res.score > best_score:
+            best_score, best_program = res.score, program
+
+        if stop_at_score is not None and best_score >= stop_at_score:
+            break
+
+        if on_candidate_evaluated is not None:
+            try:
+                on_candidate_evaluated(idx + 1, res.score, res, intervention)
+            except Exception:
+                pass
+
+    # Attach candidates for inspection
+    if hasattr(best_program, "candidate_programs"):
+        # If user object supports attribute assignment
+        try:
+            best_program.candidate_programs = sorted(candidates, key=lambda c: c.score, reverse=True)  # type: ignore[attr-defined]
+        except Exception:
+            pass
+
+    return (best_program or getattr(student, "deepcopy", student)(), records)
+
+
+__all__ = [
+    "random_search_compile",
+    "LabeledFewShot",
+    "BootstrapFewShot",
+]

synth_ai/learning/prompts/run_mipro_banking77.py

@@ -0,0 +1,160 @@
+"""
+Example: MIPROv2-style optimizer on Banking77 using Groq gpt-oss-20b.
+
+Requires:
+- .env with GROQ_API_KEY
+- datasets
+
+Run:
+- uv run -q python -m synth_ai.learning.prompts.run_mipro_banking77
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import random
+from dataclasses import dataclass, replace
+from typing import Any, Dict, List, Sequence, Tuple
+
+from dotenv import load_dotenv
+from datasets import load_dataset
+
+from synth_ai.lm.core.main_v3 import LM, build_messages
+import json
+import time
+from pathlib import Path
+from synth_ai.learning.prompts.mipro import ProgramAdapter, mipro_v2_compile, evaluate_program
+
+
+def choose_label(pred: str, label_names: List[str]) -> str:
+    norm = (pred or "").strip().lower()
+    d = {ln.lower(): ln for ln in label_names}
+    if norm in d:
+        return d[norm]
+    def score(cand: str) -> int:
+        c = cand.lower()
+        return sum(1 for w in c.split() if w in norm)
+    return max(label_names, key=score)
+
+
+def accuracy(pred: str, gold: str, labels: List[str]) -> float:
+    return 1.0 if choose_label(pred, labels) == gold else 0.0
+
+
+class NaivePromptModel:
+    """Toy prompt model that returns simple instruction variants."""
+    def generate_instructions(self, ctx: Dict[str, Any], k: int = 8) -> List[str]:
+        base = "Classify the Banking77 intent and return exactly one label."
+        variants = [
+            base,
+            base + " Be concise.",
+            base + " Use examples to guide your reasoning.",
+            base + " Return only the label text.",
+            base + " Follow the label names strictly.",
+            base + " Do not include explanations.",
+            base + " Think about similar intents before answering.",
+            base + " Carefully consider the user's message.",
+        ]
+        random.shuffle(variants)
+        return variants[:k]
+
+
+def build_run_fn(lm: LM, label_names: List[str]):
+    def run_fn(x: str, _model: Any | None = None) -> str:
+        # Use instructions and demos from adapter state (set by set_instructions/set_demos)
+        # The adapter passes state via closure; we rebuild messages here
+        instructions = state_ref.get("instructions", {}).get("main", "You are an intent classifier for Banking77.")
+        examples = "\n".join(f"Input: {a}\nLabel: {b}" for a, b in state_ref.get("demos", []))
+        sys = instructions
+        user = (f"Examples:\n{examples}\n\n" if examples else "") + f"Message: {x}\nLabel:"
+        messages = build_messages(sys, user, images_bytes=None, model_name=lm.model)
+        async def _call():
+            resp = await lm.respond_async(messages=messages)
+            return (resp.raw_response or "").strip()
+        return asyncio.run(_call())
+    return run_fn
+
+
+def set_instructions(new_instr: Dict[str, str], state: Dict[str, Any]) -> Dict[str, Any]:
+    state["instructions"] = {**state.get("instructions", {}), **new_instr}
+    return state
+
+
+def set_demos(demos: List[Tuple[str, str]], state: Dict[str, Any]) -> Dict[str, Any]:
+    state["demos"] = list(demos)
+    return state
+
+
+def main():
+    load_dotenv()
+    random.seed(0)
+
+    model = os.getenv("MODEL", "openai/gpt-oss-20b")
+    vendor = os.getenv("VENDOR", "groq")
+    lm = LM(model=model, vendor=vendor, temperature=0.0)
+
+    print("Loading Banking77 dataset (train/dev split of test for demo)...")
+    ds = load_dataset("banking77")
+    label_names: List[str] = ds["test"].features["label"].names  # type: ignore
+
+    all_items = [(r["text"], label_names[int(r["label"])]) for r in ds["test"]]
+    random.shuffle(all_items)
+    trainset: Sequence[Tuple[str, str]] = all_items[:80]
+    valset: Sequence[Tuple[str, str]] = all_items[80:160]
+
+    global state_ref
+    state_ref = {"instructions": {"main": "You are an intent classifier for Banking77."}, "demos": []}
+    adapter = ProgramAdapter(
+        run_fn=build_run_fn(lm, label_names),
+        state=state_ref,
+        _predictors=["main"],
+        set_instructions=set_instructions,
+        set_demos=set_demos,
+    )
+
+    def metric(yhat: str, y: str) -> float:
+        return accuracy(yhat, y, label_names)
+
+    prompt_model = NaivePromptModel()
+    task_model = None  # not used in this minimal example
+
+    print("Running MIPROv2-style optimizer...")
+    best, records = mipro_v2_compile(
+        student=adapter,
+        trainset=trainset,
+        valset=valset,
+        metric=metric,
+        prompt_model=prompt_model,
+        task_model=task_model,
+        max_bootstrapped_demos=6,
+        max_labeled_demos=4,
+        num_candidates=6,
+        num_trials=12,
+        minibatch=True,
+        minibatch_size=16,
+        minibatch_full_eval_steps=3,
+        seed=0,
+    )
+
+    res = evaluate_program(best, valset, metric)
+    print(f"Best program accuracy on val: {res.score:.2%} ({sum(res.subscores)}/{len(res.subscores)})")
+
+    out = {
+        "context": {
+            "model": model,
+            "vendor": vendor,
+            "train_size": len(trainset),
+            "val_size": len(valset),
+        },
+        "trials": records,
+    }
+    out_dir = Path(__file__).parent
+    fname = str(out_dir / f"mipro_banking77_{int(time.time())}.json")
+    with open(fname, "w") as f:
+        json.dump(out, f, indent=2)
+    print(f"Saved trial records to {fname}")
+
+
+if __name__ == "__main__":
+    main()