synth-ai 0.2.4.dev6__py3-none-any.whl → 0.2.4.dev8__py3-none-any.whl

synth_ai/learning/prompts/mipro.py

@@ -20,9 +20,9 @@ Notes
 from __future__ import annotations
 
 import random
+from collections.abc import Callable, Sequence
 from dataclasses import dataclass, replace
-from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Sequence, Tuple
-
+from typing import Any, Protocol
 
 # ---------------------------
 # Program adapter and protocols
@@ -36,16 +36,16 @@ class PredictProgram(Protocol):
     by wrapping it with `ProgramAdapter` below.
     """
 
-    def deepcopy(self) -> "PredictProgram": ...
+    def deepcopy(self) -> PredictProgram: ...
 
-    def run(self, x: Any, *, model: Optional[Any] = None) -> Any: ...
+    def run(self, x: Any, *, model: Any | None = None) -> Any: ...
 
-    def with_instructions(self, instructions: Dict[str, str]) -> "PredictProgram": ...
+    def with_instructions(self, instructions: dict[str, str]) -> PredictProgram: ...
 
-    def with_demos(self, demos: List[Tuple[Any, Any]]) -> "PredictProgram": ...
+    def with_demos(self, demos: list[tuple[Any, Any]]) -> PredictProgram: ...
 
     @property
-    def predictors(self) -> List[str]: ...
+    def predictors(self) -> list[str]: ...
 
 
 @dataclass
@@ -59,28 +59,28 @@ class ProgramAdapter:
     - set_demos: Callable to update demos (global or per predictor)
     """
 
-    run_fn: Callable[[Any, Optional[Any]], Any]
-    state: Dict[str, Any]
-    _predictors: List[str]
-    set_instructions: Callable[[Dict[str, str], Dict[str, Any]], Dict[str, Any]]
-    set_demos: Callable[[List[Tuple[Any, Any]], Dict[str, Any]], Dict[str, Any]]
+    run_fn: Callable[[Any, Any | None], Any]
+    state: dict[str, Any]
+    _predictors: list[str]
+    set_instructions: Callable[[dict[str, str], dict[str, Any]], dict[str, Any]]
+    set_demos: Callable[[list[tuple[Any, Any]], dict[str, Any]], dict[str, Any]]
 
-    def deepcopy(self) -> "ProgramAdapter":
+    def deepcopy(self) -> ProgramAdapter:
         return replace(self, state={**self.state})
 
-    def run(self, x: Any, *, model: Optional[Any] = None) -> Any:
+    def run(self, x: Any, *, model: Any | None = None) -> Any:
         return self.run_fn(x, model)
 
-    def with_instructions(self, instructions: Dict[str, str]) -> "ProgramAdapter":
+    def with_instructions(self, instructions: dict[str, str]) -> ProgramAdapter:
         new_state = self.set_instructions(instructions, {**self.state})
         return replace(self, state=new_state)
 
-    def with_demos(self, demos: List[Tuple[Any, Any]]) -> "ProgramAdapter":
+    def with_demos(self, demos: list[tuple[Any, Any]]) -> ProgramAdapter:
         new_state = self.set_demos(demos, {**self.state})
         return replace(self, state=new_state)
 
     @property
-    def predictors(self) -> List[str]:
+    def predictors(self) -> list[str]:
         return list(self._predictors)
 
 
@@ -89,9 +89,11 @@ class ProgramAdapter:
 # ---------------------------
 
 
-def summarize_dataset(trainset: Sequence[Tuple[Any, Any]], max_items: int = 50) -> str:
+def summarize_dataset(trainset: Sequence[tuple[Any, Any]], max_items: int = 50) -> str:
     n = len(trainset)
-    ex = ", ".join(repr(trainset[i][0])[:40] for i in range(0, min(max_items, n), max(1, n // max_items or 1)))
+    ex = ", ".join(
+        repr(trainset[i][0])[:40] for i in range(0, min(max_items, n), max(1, n // max_items or 1))
+    )
     return f"Dataset size: {n}. Example inputs: {ex}"
 
 
@@ -109,7 +111,7 @@ def random_tip(rng: random.Random) -> str:
     return rng.choice(tips)
 
 
-def choose(items: Sequence[Any], rng: Optional[random.Random] = None) -> Any:
+def choose(items: Sequence[Any], rng: random.Random | None = None) -> Any:
     r = rng or random
     return r.choice(items)
 
@@ -122,10 +124,12 @@ def choose(items: Sequence[Any], rng: Optional[random.Random] = None) -> Any:
 @dataclass
 class EvalResult:
     score: float
-    subscores: List[float]
+    subscores: list[float]
 
 
-def evaluate_program(program: PredictProgram, dataset: Sequence[Tuple[Any, Any]], metric: Callable[[Any, Any], float]) -> EvalResult:
+def evaluate_program(
+    program: PredictProgram, dataset: Sequence[tuple[Any, Any]], metric: Callable[[Any, Any], float]
+) -> EvalResult:
     subs = []
     for x, y in dataset:
         yhat = program.run(x)
@@ -140,8 +144,8 @@ def evaluate_program(program: PredictProgram, dataset: Sequence[Tuple[Any, Any]]
 
 def mipro_v2_compile(
     student: PredictProgram,
-    trainset: Sequence[Tuple[Any, Any]],
-    valset: Sequence[Tuple[Any, Any]],
+    trainset: Sequence[tuple[Any, Any]],
+    valset: Sequence[tuple[Any, Any]],
     metric: Callable[[Any, Any], float],
     *,
     prompt_model: Any,
@@ -159,7 +163,7 @@ def mipro_v2_compile(
     data_aware: bool = True,
     tip_aware: bool = True,
     fewshot_aware: bool = True,
-) -> Tuple[PredictProgram, List[Dict[str, Any]]]:
+) -> tuple[PredictProgram, list[dict[str, Any]]]:
     """MIPROv2-style optimizer.
 
     Arguments mirror the DSPy pseudocode but remain provider-agnostic. The
@@ -171,9 +175,9 @@ def mipro_v2_compile(
     program = student.deepcopy()
 
     # Step 1: bootstrap few-shot example candidates
-    demo_candidates: List[Dict[str, Any]] = []
+    demo_candidates: list[dict[str, Any]] = []
     for _ in range(num_candidates):
-        boot: List[Tuple[Any, Any]] = []
+        boot: list[tuple[Any, Any]] = []
         # collect bootstrapped, self-consistent demos
         while len(boot) < max_bootstrapped_demos:
             x, y = rng.choice(trainset)
@@ -184,9 +188,9 @@ def mipro_v2_compile(
         demo_candidates.append({"boot": boot, "labeled": labeled})
 
     # Step 2: propose instruction candidates per predictor
-    instr_candidates: Dict[str, List[str]] = {}
-    for pred in (program.predictors or ["predictor"]):
-        ctx: Dict[str, Any] = {}
+    instr_candidates: dict[str, list[str]] = {}
+    for pred in program.predictors or ["predictor"]:
+        ctx: dict[str, Any] = {}
         if data_aware:
             ctx["dataset_summary"] = summarize_dataset(trainset)
         if program_aware:
@@ -199,12 +203,12 @@ def mipro_v2_compile(
         instr_candidates[pred] = list(cand)
 
     # Step 3: Bayesian-optimization-like search (random proposer placeholder)
-    history: List[Tuple[Dict[str, Any], float]] = []
-    records: List[Dict[str, Any]] = []
+    history: list[tuple[dict[str, Any], float]] = []
+    records: list[dict[str, Any]] = []
     best_score = -1.0
-    best_cfg: Optional[Dict[str, Any]] = None
+    best_cfg: dict[str, Any] | None = None
 
-    def propose(history_: List[Tuple[Dict[str, Any], float]]) -> Dict[str, Any]:
+    def propose(history_: list[tuple[dict[str, Any], float]]) -> dict[str, Any]:
         # Placeholder: randomly sample from the cartesian product
         instructions = {pred: choose(instr_candidates[pred], rng) for pred in instr_candidates}
         demos = choose(demo_candidates, rng) if demo_candidates else None
@@ -227,15 +231,17 @@ def mipro_v2_compile(
         batch_res = evaluate_program(program_t, batch, metric)
         s_t = batch_res.score
         history.append((theta, s_t))
-        records.append({
-            "trial": t,
-            "evaluation": "batch" if minibatch else "full",
-            "score": s_t,
-            "intervention": {
-                "instructions": theta.get("instructions"),
-                "demo_set": theta.get("demo_set"),
-            },
-        })
+        records.append(
+            {
+                "trial": t,
+                "evaluation": "batch" if minibatch else "full",
+                "score": s_t,
+                "intervention": {
+                    "instructions": theta.get("instructions"),
+                    "demo_set": theta.get("demo_set"),
+                },
+            }
+        )
 
         if (not minibatch) or (t % max(1, minibatch_full_eval_steps) == 0):
             full_res = evaluate_program(program_t, valset, metric)
@@ -243,15 +249,17 @@ def mipro_v2_compile(
             if s_full > best_score:
                 best_score = s_full
                 best_cfg = theta
-            records.append({
-                "trial": t,
-                "evaluation": "full",
-                "score": s_full,
-                "intervention": {
-                    "instructions": theta.get("instructions"),
-                    "demo_set": theta.get("demo_set"),
-                },
-            })
+            records.append(
+                {
+                    "trial": t,
+                    "evaluation": "full",
+                    "score": s_full,
+                    "intervention": {
+                        "instructions": theta.get("instructions"),
+                        "demo_set": theta.get("demo_set"),
+                    },
+                }
+            )
 
     if best_cfg is None:
         return program, records
@@ -275,6 +283,7 @@ __all__ = [
 class ExampleTwoStepDag:
     pass
 
+
 """
 A -> B
 """

synth_ai/learning/prompts/random_search.py

@@ -8,10 +8,11 @@ metric, and this module will explore baselines and bootstrapped few-shot variant
 
 from __future__ import annotations
 
+import contextlib
 import random
+from collections.abc import Callable, Sequence
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
-
+from typing import Any
 
 # ---------------------------
 # Protocol-like expectations (duck-typed)
@@ -25,7 +26,7 @@ class _ProgramLike:
     def deepcopy(self):  # deep copy
         return self
 
-    def with_demos(self, demos: List[Tuple[Any, Any]]):
+    def with_demos(self, demos: list[tuple[Any, Any]]):
         return self
 
     def run(self, x: Any) -> Any:
@@ -40,10 +41,12 @@ class _ProgramLike:
 @dataclass
 class EvalResult:
     score: float
-    subscores: List[float]
+    subscores: list[float]
 
 
-def evaluate(program: _ProgramLike, dataset: Sequence[Tuple[Any, Any]], metric: Callable[[Any, Any], float]) -> EvalResult:
+def evaluate(
+    program: _ProgramLike, dataset: Sequence[tuple[Any, Any]], metric: Callable[[Any, Any], float]
+) -> EvalResult:
     subs = []
     for x, y in dataset:
         subs.append(metric(program.run(x), y))
@@ -54,7 +57,9 @@ class LabeledFewShot:
     def __init__(self, k: int):
         self.k = k
 
-    def compile(self, student: _ProgramLike, trainset: Sequence[Tuple[Any, Any]], sample: bool = True) -> _ProgramLike:
+    def compile(
+        self, student: _ProgramLike, trainset: Sequence[tuple[Any, Any]], sample: bool = True
+    ) -> _ProgramLike:
         p = getattr(student, "deepcopy", student.reset_copy)()
         demos = list(trainset)
         if sample:
@@ -68,10 +73,10 @@ class BootstrapFewShot:
         self,
         *,
         metric: Callable[[Any, Any], float],
-        metric_threshold: Optional[float] = None,
+        metric_threshold: float | None = None,
         max_bootstrapped_demos: int = 8,
         max_labeled_demos: int = 0,
-        teacher_settings: Optional[Dict[str, Any]] = None,
+        teacher_settings: dict[str, Any] | None = None,
         max_rounds: int = 1,
     ):
         self.metric = metric
@@ -84,18 +89,18 @@ class BootstrapFewShot:
     def compile(
         self,
         student: _ProgramLike,
-        teacher: Optional[_ProgramLike],
-        trainset: Sequence[Tuple[Any, Any]],
+        teacher: _ProgramLike | None,
+        trainset: Sequence[tuple[Any, Any]],
     ) -> _ProgramLike:
         p = getattr(student, "deepcopy", student.reset_copy)()
         rng = random.Random()
         # If bootstrapped demos disabled, return labeled-only few-shot quickly
         if self.max_bootstrapped_demos <= 0:
-            demos: List[Tuple[Any, Any]] = []
+            demos: list[tuple[Any, Any]] = []
             if self.max_labeled_demos > 0:
                 demos += rng.sample(list(trainset), k=min(self.max_labeled_demos, len(trainset)))
             return p.with_demos(demos)
-        boot: List[Tuple[Any, Any]] = []
+        boot: list[tuple[Any, Any]] = []
         # Bootstrap demos by self consistency
         for _ in range(self.max_rounds):
             rng.shuffle(trainset := list(trainset))
@@ -127,33 +132,29 @@ class BootstrapFewShot:
 @dataclass
 class Candidate:
     score: float
-    subscores: List[float]
+    subscores: list[float]
     seed: int
     program: _ProgramLike
 
 
 def random_search_compile(
     student: _ProgramLike,
-    trainset: Sequence[Tuple[Any, Any]],
-    valset: Sequence[Tuple[Any, Any]],
+    trainset: Sequence[tuple[Any, Any]],
+    valset: Sequence[tuple[Any, Any]],
     metric: Callable[[Any, Any], float],
     *,
     max_bootstrapped_demos: int = 8,
     max_labeled_demos: int = 4,
     max_rounds: int = 2,
     num_candidate_programs: int = 16,
-    stop_at_score: Optional[float] = None,
-    evaluate_fn: Optional[Callable[[
-        _ProgramLike,
-        Sequence[Tuple[Any, Any]],
-        Callable[[Any, Any], float]
-    ], EvalResult]] = None,
-    on_candidate_evaluated: Optional[Callable[[int, float, EvalResult, Dict[str, Any]], None]] = None,
-) -> Tuple[_ProgramLike, List[Dict[str, Any]]]:
-    best_program: Optional[_ProgramLike] = None
+    stop_at_score: float | None = None,
+    evaluate_fn: Callable[[_ProgramLike, Sequence[tuple[Any, Any]], Callable[[Any, Any], float]], EvalResult] | None = None,
+    on_candidate_evaluated: Callable[[int, float, EvalResult, dict[str, Any]], None] | None = None,
+) -> tuple[_ProgramLike, list[dict[str, Any]]]:
+    best_program: _ProgramLike | None = None
     best_score = float("-inf")
-    candidates: List[Candidate] = []
-    records: List[Dict[str, Any]] = []
+    candidates: list[Candidate] = []
+    records: list[dict[str, Any]] = []
 
     seeds = list(range(num_candidate_programs))
     seeds = [-3, -2, -1] + seeds  # zero-shot, labeled few-shot, bootstrapped few-shot
@@ -174,7 +175,9 @@ def random_search_compile(
             if max_bootstrapped_demos <= 0:
                 size = 0
             else:
-                size = max_bootstrapped_demos if seed == -1 else rng.randint(1, max_bootstrapped_demos)
+                size = (
+                    max_bootstrapped_demos if seed == -1 else rng.randint(1, max_bootstrapped_demos)
+                )
             program = BootstrapFewShot(
                 metric=metric,
                 metric_threshold=None,
@@ -184,14 +187,18 @@ def random_search_compile(
                 max_rounds=max_rounds,
             ).compile(student, teacher=None, trainset=train_copy)
 
-        res = (evaluate_fn(program, valset, metric) if evaluate_fn else evaluate(program, valset, metric))
+        res = (
+            evaluate_fn(program, valset, metric)
+            if evaluate_fn
+            else evaluate(program, valset, metric)
+        )
         cand = Candidate(score=res.score, subscores=res.subscores, seed=seed, program=program)
         candidates.append(cand)
         # Record an intervention summary for reproducibility
-        intervention: Dict[str, Any] = {"seed": seed}
+        intervention: dict[str, Any] = {"seed": seed}
         if hasattr(program, "demos"):
             try:
-                intervention["demos"] = getattr(program, "demos")  # type: ignore
+                intervention["demos"] = program.demos  # type: ignore
             except Exception:
                 intervention["demos"] = None
         # Type of candidate
@@ -203,12 +210,6 @@ def random_search_compile(
             intervention["label"] = f"labeled-{max_labeled_demos}"
         else:
             intervention["kind"] = "bootstrapped_few_shot"
-            bs = 0
-            try:
-                # try to infer from program demos length if present
-                bs = len(intervention.get("demos") or [])
-            except Exception:
-                bs = 0
             intervention["label"] = f"boot-b{max_bootstrapped_demos}-l{max_labeled_demos}"
         record_obj = {
             "score": cand.score,
@@ -224,18 +225,16 @@ def random_search_compile(
             break
 
         if on_candidate_evaluated is not None:
-            try:
+            with contextlib.suppress(Exception):
                 on_candidate_evaluated(idx + 1, res.score, res, intervention)
-            except Exception:
-                pass
 
     # Attach candidates for inspection
     if hasattr(best_program, "candidate_programs"):
         # If user object supports attribute assignment
-        try:
-            best_program.candidate_programs = sorted(candidates, key=lambda c: c.score, reverse=True)  # type: ignore[attr-defined]
-        except Exception:
-            pass
+        with contextlib.suppress(Exception):
+            best_program.candidate_programs = sorted(
+                candidates, key=lambda c: c.score, reverse=True
+            )  # type: ignore[attr-defined]
 
     return (best_program or getattr(student, "deepcopy", student)(), records)
 

synth_ai/learning/prompts/run_mipro_banking77.py

@@ -12,39 +12,41 @@ Run:
 from __future__ import annotations
 
 import asyncio
+import json
 import os
 import random
-from dataclasses import dataclass, replace
-from typing import Any, Dict, List, Sequence, Tuple
+import time
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
 
-from dotenv import load_dotenv
 from datasets import load_dataset
-
+from dotenv import load_dotenv
+from synth_ai.learning.prompts.mipro import ProgramAdapter, evaluate_program, mipro_v2_compile
 from synth_ai.lm.core.main_v3 import LM, build_messages
-import json
-import time
-from pathlib import Path
-from synth_ai.learning.prompts.mipro import ProgramAdapter, mipro_v2_compile, evaluate_program
 
 
-def choose_label(pred: str, label_names: List[str]) -> str:
+def choose_label(pred: str, label_names: list[str]) -> str:
     norm = (pred or "").strip().lower()
     d = {ln.lower(): ln for ln in label_names}
     if norm in d:
         return d[norm]
+
     def score(cand: str) -> int:
         c = cand.lower()
         return sum(1 for w in c.split() if w in norm)
+
     return max(label_names, key=score)
 
 
-def accuracy(pred: str, gold: str, labels: List[str]) -> float:
+def accuracy(pred: str, gold: str, labels: list[str]) -> float:
     return 1.0 if choose_label(pred, labels) == gold else 0.0
 
 
 class NaivePromptModel:
     """Toy prompt model that returns simple instruction variants."""
-    def generate_instructions(self, ctx: Dict[str, Any], k: int = 8) -> List[str]:
+
+    def generate_instructions(self, ctx: dict[str, Any], k: int = 8) -> list[str]:
         base = "Classify the Banking77 intent and return exactly one label."
         variants = [
             base,
@@ -60,28 +62,33 @@ class NaivePromptModel:
         return variants[:k]
 
 
-def build_run_fn(lm: LM, label_names: List[str]):
+def build_run_fn(lm: LM, label_names: list[str]):
     def run_fn(x: str, _model: Any | None = None) -> str:
         # Use instructions and demos from adapter state (set by set_instructions/set_demos)
         # The adapter passes state via closure; we rebuild messages here
-        instructions = state_ref.get("instructions", {}).get("main", "You are an intent classifier for Banking77.")
+        instructions = state_ref.get("instructions", {}).get(
+            "main", "You are an intent classifier for Banking77."
+        )
         examples = "\n".join(f"Input: {a}\nLabel: {b}" for a, b in state_ref.get("demos", []))
         sys = instructions
         user = (f"Examples:\n{examples}\n\n" if examples else "") + f"Message: {x}\nLabel:"
         messages = build_messages(sys, user, images_bytes=None, model_name=lm.model)
+
         async def _call():
             resp = await lm.respond_async(messages=messages)
             return (resp.raw_response or "").strip()
+
         return asyncio.run(_call())
+
     return run_fn
 
 
-def set_instructions(new_instr: Dict[str, str], state: Dict[str, Any]) -> Dict[str, Any]:
+def set_instructions(new_instr: dict[str, str], state: dict[str, Any]) -> dict[str, Any]:
     state["instructions"] = {**state.get("instructions", {}), **new_instr}
     return state
 
 
-def set_demos(demos: List[Tuple[str, str]], state: Dict[str, Any]) -> Dict[str, Any]:
+def set_demos(demos: list[tuple[str, str]], state: dict[str, Any]) -> dict[str, Any]:
     state["demos"] = list(demos)
     return state
 
@@ -96,15 +103,18 @@ def main():
 
     print("Loading Banking77 dataset (train/dev split of test for demo)...")
     ds = load_dataset("banking77")
-    label_names: List[str] = ds["test"].features["label"].names  # type: ignore
+    label_names: list[str] = ds["test"].features["label"].names  # type: ignore
 
     all_items = [(r["text"], label_names[int(r["label"])]) for r in ds["test"]]
     random.shuffle(all_items)
-    trainset: Sequence[Tuple[str, str]] = all_items[:80]
-    valset: Sequence[Tuple[str, str]] = all_items[80:160]
+    trainset: Sequence[tuple[str, str]] = all_items[:80]
+    valset: Sequence[tuple[str, str]] = all_items[80:160]
 
     global state_ref
-    state_ref = {"instructions": {"main": "You are an intent classifier for Banking77."}, "demos": []}
+    state_ref = {
+        "instructions": {"main": "You are an intent classifier for Banking77."},
+        "demos": [],
+    }
     adapter = ProgramAdapter(
         run_fn=build_run_fn(lm, label_names),
         state=state_ref,
@@ -138,7 +148,9 @@ def main():
     )
 
     res = evaluate_program(best, valset, metric)
-    print(f"Best program accuracy on val: {res.score:.2%} ({sum(res.subscores)}/{len(res.subscores)})")
+    print(
+        f"Best program accuracy on val: {res.score:.2%} ({sum(res.subscores)}/{len(res.subscores)})"
+    )
 
     out = {
         "context": {