synth-ai 0.2.4.dev6__py3-none-any.whl → 0.2.4.dev7__py3-none-any.whl

synth_ai/environments/tasks/filters.py

@@ -1,6 +1,8 @@
-from typing import Any, Collection, Optional
+from collections.abc import Collection
 from dataclasses import dataclass
-from synth_ai.environments.tasks.core import TaskInstanceMetadataFilter, TaskInstance
+from typing import Any
+
+from synth_ai.environments.tasks.core import TaskInstance, TaskInstanceMetadataFilter
 
 
 @dataclass
@@ -18,8 +20,8 @@ class ValueFilter(TaskInstanceMetadataFilter):
 @dataclass
 class RangeFilter(TaskInstanceMetadataFilter):
     key: str
-    min_val: Optional[float] = None
-    max_val: Optional[float] = None
+    min_val: float | None = None
+    max_val: float | None = None
 
     def __call__(self, instance: TaskInstance) -> bool:
         instance_value = getattr(instance.metadata, self.key, None)

synth_ai/environments/tasks/utils.py

@@ -2,17 +2,19 @@
 Utility functions and generic filters for taskset creation.
 """
 
-from typing import Any, Collection, Optional, List, Set
+from collections.abc import Collection
+from typing import Any
 from uuid import UUID, uuid4
+
 from synth_ai.environments.tasks.core import (
-    TaskInstanceMetadataFilter,
-    TaskInstanceSet,
     SplitInfo,
     TaskInstance,
+    TaskInstanceMetadataFilter,
+    TaskInstanceSet,
 )
 
 
-def parse_or_new_uuid(raw_id: Optional[str]) -> UUID:
+def parse_or_new_uuid(raw_id: str | None) -> UUID:
     """
     Parse a raw ID string into a UUID, or generate a new one if invalid or missing.
     """
@@ -43,8 +45,8 @@ class RangeFilter(TaskInstanceMetadataFilter):
     def __init__(
         self,
         key: str,
-        min_value: Optional[float] = None,
-        max_value: Optional[float] = None,
+        min_value: float | None = None,
+        max_value: float | None = None,
     ):
         self.key = key
         self.min_value = min_value
@@ -62,15 +64,15 @@ class RangeFilter(TaskInstanceMetadataFilter):
 def make_taskset(
     name: str,
     description: str,
-    instances: List[TaskInstance],
-    val_filter: Optional[TaskInstanceMetadataFilter] = None,
-    test_filter: Optional[TaskInstanceMetadataFilter] = None,
+    instances: list[TaskInstance],
+    val_filter: TaskInstanceMetadataFilter | None = None,
+    test_filter: TaskInstanceMetadataFilter | None = None,
 ) -> TaskInstanceSet:
     """
     Assemble a TaskInstanceSet by applying optional validation and test filters.
     """
-    val_ids: Set[Any] = set()
-    test_ids: Set[Any] = set()
+    val_ids: set[Any] = set()
+    test_ids: set[Any] = set()
     if val_filter:
         val_ids = {inst.id for inst in instances if val_filter(inst)}
     if test_filter:

synth_ai/evals/base.py

@@ -1,9 +1,8 @@
-from typing import List
 
 
 class Judgement:
     def __init__(
-        self, criteria: str, score: float, reasoning: str = "", evidence: List[str] = None
+        self, criteria: str, score: float, reasoning: str = "", evidence: list[str] = None
     ):
         self.criteria = criteria
         self.score = score
@@ -12,5 +11,5 @@ class Judgement:
 
 
 class BaseEval:
-    async def run(self, data: any) -> List[Judgement]:
+    async def run(self, data: any) -> list[Judgement]:
         pass

synth_ai/experimental/synth_oss.py

@@ -1,5 +1,5 @@
-
-"""
+# ruff: noqa
+'''
 Synth OSS Integration Module
 
 This module provides integration with Synth's open-source inference and training APIs
@@ -336,7 +336,7 @@ Implementation sketch (backend == "synth")
 The method is a *no-op* for the default (OpenAI) backend so existing code keeps
 working.
 
-"""
+'''
 
 
 """
@@ -443,4 +443,4 @@ async def warmup(
 So: **the existing endpoint does not yet support GPU selection; we need to add
 the small change above on the `learning_v2` side and then LM.warmup can request
 specific GPUs.**
-"""
+"""

synth_ai/learning/gateway.py

@@ -1,4 +1,2 @@
-
-
 class OfflineGateway:
-    pass
+    pass

synth_ai/learning/prompts/banking77_injection_eval.py

@@ -18,16 +18,15 @@ from __future__ import annotations
 import asyncio
 import os
 import random
-from typing import List, Dict, Any, Tuple
+from typing import Any
 
-from dotenv import load_dotenv
 from datasets import load_dataset
-
+from dotenv import load_dotenv
 from synth_ai.lm.core.main_v3 import LM, build_messages
 from synth_ai.lm.overrides import LMOverridesContext
 
 
-async def classify_one(lm: LM, text: str, label_names: List[str]) -> str:
+async def classify_one(lm: LM, text: str, label_names: list[str]) -> str:
     labels_joined = ", ".join(label_names)
     system_message = (
         "You are an intent classifier for the Banking77 dataset. "
@@ -41,7 +40,7 @@ async def classify_one(lm: LM, text: str, label_names: List[str]) -> str:
     return (resp.raw_response or "").strip()
 
 
-def choose_label(pred: str, label_names: List[str]) -> str:
+def choose_label(pred: str, label_names: list[str]) -> str:
     norm_pred = pred.strip().lower()
     label_lookup = {ln.lower(): ln for ln in label_names}
     mapped = label_lookup.get(norm_pred)
@@ -56,12 +55,18 @@ def choose_label(pred: str, label_names: List[str]) -> str:
     return max(label_names, key=score)
 
 
-async def eval_context(lm: LM, items: List[Tuple[str, str]], label_names: List[str], ctx_name: str, specs: List[Dict[str, Any]]) -> Tuple[str, int, int]:
+async def eval_context(
+    lm: LM,
+    items: list[tuple[str, str]],
+    label_names: list[str],
+    ctx_name: str,
+    specs: list[dict[str, Any]],
+) -> tuple[str, int, int]:
     correct = 0
     with LMOverridesContext(specs):
         tasks = [classify_one(lm, text, label_names) for text, _ in items]
         results = await asyncio.gather(*tasks, return_exceptions=True)
-    for (text, gold), pred in zip(items, results):
+    for (text, gold), pred in zip(items, results, strict=False):
         if isinstance(pred, Exception):
             # Treat exceptions as incorrect
             continue
@@ -81,7 +86,7 @@ async def main() -> None:
 
     print("Loading Banking77 dataset (split='test')...")
     ds = load_dataset("banking77", split="test")
-    label_names: List[str] = ds.features["label"].names  # type: ignore
+    label_names: list[str] = ds.features["label"].names  # type: ignore
 
     idxs = random.sample(range(len(ds)), k=min(n, len(ds)))
     items = [
@@ -90,7 +95,7 @@ async def main() -> None:
     ]
 
     # Define a few override contexts to compare
-    contexts: List[Dict[str, Any]] = [
+    contexts: list[dict[str, Any]] = [
         {
             "name": "baseline (no overrides)",
             "overrides": [],
@@ -145,7 +150,7 @@ async def main() -> None:
     print(f"\nEvaluating {len(contexts)} contexts on {len(items)} Banking77 samples (async)...")
 
     # Evaluate each context sequentially but batched (each context classifies in parallel)
-    results: List[Tuple[str, int, int]] = []
+    results: list[tuple[str, int, int]] = []
     for ctx in contexts:
         name = ctx["name"]
         specs = ctx["overrides"]

synth_ai/learning/prompts/hello_world_in_context_injection_ex.py

@@ -27,18 +27,17 @@ from __future__ import annotations
 import asyncio
 import os
 import random
-from typing import Any, Dict, List, Optional
 
 from datasets import load_dataset
 
 # Use the v3 LM class present in this repo
 from synth_ai.lm.core.main_v3 import LM, build_messages
-from synth_ai.tracing_v3.session_tracer import SessionTracer
-from synth_ai.tracing_v3.abstractions import LMCAISEvent
-
 
 # Use Overrides context to demonstrate matching by content
 from synth_ai.lm.overrides import LMOverridesContext
+from synth_ai.tracing_v3.abstractions import LMCAISEvent
+from synth_ai.tracing_v3.session_tracer import SessionTracer
+
 INJECTION_RULES = [
     {"find": "accnt", "replace": "account"},
     {"find": "atm", "replace": "ATM"},
@@ -46,7 +45,7 @@ INJECTION_RULES = [
 ]
 
 
-async def classify_sample(lm: LM, text: str, label_names: List[str]) -> str:
+async def classify_sample(lm: LM, text: str, label_names: list[str]) -> str:
     """Classify one Banking77 utterance and return the predicted label name."""
     labels_joined = ", ".join(label_names)
     system_message = (
@@ -77,7 +76,7 @@ async def main() -> None:
     # Columns: {"text": str, "label": int}; label names at ds.features["label"].names
     print("Loading Banking77 dataset (split='test')...")
     ds = load_dataset("banking77", split="test")
-    label_names: List[str] = ds.features["label"].names  # type: ignore
+    label_names: list[str] = ds.features["label"].names  # type: ignore
 
     # Sample a few items for a quick demo
     n = int(os.getenv("N_SAMPLES", "8"))
@@ -116,7 +115,9 @@ async def main() -> None:
 
             is_correct = pred_label == gold_label
             correct += int(is_correct)
-            print(f"[{i}] text={text!r}\n    gold={gold_label}\n    pred={pred} -> mapped={pred_label} {'✅' if is_correct else '❌'}")
+            print(
+                f"[{i}] text={text!r}\n    gold={gold_label}\n    pred={pred} -> mapped={pred_label} {'✅' if is_correct else '❌'}"
+            )
 
     if idxs:
         acc = correct / len(idxs)
@@ -137,7 +138,11 @@ async def main() -> None:
     with LMOverridesContext([{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]):
         _ = await classify_sample(lm_traced, test_text, label_names)
     # inspect trace
-    events = [e for e in (tracer.current_session.event_history if tracer.current_session else []) if isinstance(e, LMCAISEvent)]
+    events = [
+        e
+        for e in (tracer.current_session.event_history if tracer.current_session else [])
+        if isinstance(e, LMCAISEvent)
+    ]
     assert events, "No LMCAISEvent recorded by SessionTracer"
     cr = events[-1].call_records[0]
     traced_user = ""
@@ -145,7 +150,7 @@ async def main() -> None:
         if m.role == "user":
             for part in m.parts:
                 if getattr(part, "type", None) == "text":
-                    traced_user += (part.text or "")
+                    traced_user += part.text or ""
     assert "ATM" in traced_user, f"Expected substitution in traced prompt; got: {traced_user!r}"
     print("LM path trace verified: substitution present in traced prompt.")
     await tracer.end_timestep()
@@ -155,7 +160,7 @@ async def main() -> None:
     try:
         import synth_ai.lm.provider_support.openai as _synth_openai_patch  # noqa: F401
         from openai import AsyncOpenAI
-        from datasets import load_dataset as _ld  # ensure datasets present
+
         base_url = os.getenv("OPENAI_BASE_URL", "https://api.groq.com/openai/v1")
         api_key = os.getenv("OPENAI_API_KEY") or os.getenv("GROQ_API_KEY") or ""
         client = AsyncOpenAI(base_url=base_url, api_key=api_key)
@@ -163,8 +168,12 @@ async def main() -> None:
             {"role": "system", "content": "Echo user label."},
             {"role": "user", "content": f"Please classify: {test_text}"},
         ]
-        with LMOverridesContext([{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]):
-            resp = await client.chat.completions.create(model=model, messages=messages, temperature=0)
+        with LMOverridesContext(
+            [{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]
+        ):
+            _ = await client.chat.completions.create(
+                model=model, messages=messages, temperature=0
+            )
         # Not all models echo input; instead, verify that our injected expectation matches
         expected_user = messages[1]["content"].replace("atm", "ATM")
         if messages[1]["content"] == expected_user:
@@ -176,13 +185,16 @@ async def main() -> None:
 
     # 3) Anthropic wrapper path (AsyncClient): ensure apply_injection is active
     try:
-        import synth_ai.lm.provider_support.anthropic as _synth_anthropic_patch  # noqa: F401
         import anthropic
+        import synth_ai.lm.provider_support.anthropic as _synth_anthropic_patch  # noqa: F401
+
         a_model = os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-20241022")
         a_key = os.getenv("ANTHROPIC_API_KEY")
         if a_key:
             a_client = anthropic.AsyncClient(api_key=a_key)
-            with LMOverridesContext([{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]):
+            with LMOverridesContext(
+                [{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]
+            ):
                 _ = await a_client.messages.create(
                     model=a_model,
                     system="Echo user label.",

synth_ai/learning/prompts/mipro.py

@@ -20,9 +20,9 @@ Notes
 from __future__ import annotations
 
 import random
+from collections.abc import Callable, Sequence
 from dataclasses import dataclass, replace
-from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Sequence, Tuple
-
+from typing import Any, Protocol
 
 # ---------------------------
 # Program adapter and protocols
@@ -36,16 +36,16 @@ class PredictProgram(Protocol):
     by wrapping it with `ProgramAdapter` below.
     """
 
-    def deepcopy(self) -> "PredictProgram": ...
+    def deepcopy(self) -> PredictProgram: ...
 
-    def run(self, x: Any, *, model: Optional[Any] = None) -> Any: ...
+    def run(self, x: Any, *, model: Any | None = None) -> Any: ...
 
-    def with_instructions(self, instructions: Dict[str, str]) -> "PredictProgram": ...
+    def with_instructions(self, instructions: dict[str, str]) -> PredictProgram: ...
 
-    def with_demos(self, demos: List[Tuple[Any, Any]]) -> "PredictProgram": ...
+    def with_demos(self, demos: list[tuple[Any, Any]]) -> PredictProgram: ...
 
     @property
-    def predictors(self) -> List[str]: ...
+    def predictors(self) -> list[str]: ...
 
 
 @dataclass
@@ -59,28 +59,28 @@ class ProgramAdapter:
     - set_demos: Callable to update demos (global or per predictor)
     """
 
-    run_fn: Callable[[Any, Optional[Any]], Any]
-    state: Dict[str, Any]
-    _predictors: List[str]
-    set_instructions: Callable[[Dict[str, str], Dict[str, Any]], Dict[str, Any]]
-    set_demos: Callable[[List[Tuple[Any, Any]], Dict[str, Any]], Dict[str, Any]]
+    run_fn: Callable[[Any, Any | None], Any]
+    state: dict[str, Any]
+    _predictors: list[str]
+    set_instructions: Callable[[dict[str, str], dict[str, Any]], dict[str, Any]]
+    set_demos: Callable[[list[tuple[Any, Any]], dict[str, Any]], dict[str, Any]]
 
-    def deepcopy(self) -> "ProgramAdapter":
+    def deepcopy(self) -> ProgramAdapter:
         return replace(self, state={**self.state})
 
-    def run(self, x: Any, *, model: Optional[Any] = None) -> Any:
+    def run(self, x: Any, *, model: Any | None = None) -> Any:
         return self.run_fn(x, model)
 
-    def with_instructions(self, instructions: Dict[str, str]) -> "ProgramAdapter":
+    def with_instructions(self, instructions: dict[str, str]) -> ProgramAdapter:
         new_state = self.set_instructions(instructions, {**self.state})
         return replace(self, state=new_state)
 
-    def with_demos(self, demos: List[Tuple[Any, Any]]) -> "ProgramAdapter":
+    def with_demos(self, demos: list[tuple[Any, Any]]) -> ProgramAdapter:
         new_state = self.set_demos(demos, {**self.state})
         return replace(self, state=new_state)
 
     @property
-    def predictors(self) -> List[str]:
+    def predictors(self) -> list[str]:
         return list(self._predictors)
 
 
@@ -89,9 +89,11 @@ class ProgramAdapter:
 # ---------------------------
 
 
-def summarize_dataset(trainset: Sequence[Tuple[Any, Any]], max_items: int = 50) -> str:
+def summarize_dataset(trainset: Sequence[tuple[Any, Any]], max_items: int = 50) -> str:
     n = len(trainset)
-    ex = ", ".join(repr(trainset[i][0])[:40] for i in range(0, min(max_items, n), max(1, n // max_items or 1)))
+    ex = ", ".join(
+        repr(trainset[i][0])[:40] for i in range(0, min(max_items, n), max(1, n // max_items or 1))
+    )
     return f"Dataset size: {n}. Example inputs: {ex}"
 
 
@@ -109,7 +111,7 @@ def random_tip(rng: random.Random) -> str:
     return rng.choice(tips)
 
 
-def choose(items: Sequence[Any], rng: Optional[random.Random] = None) -> Any:
+def choose(items: Sequence[Any], rng: random.Random | None = None) -> Any:
     r = rng or random
     return r.choice(items)
 
@@ -122,10 +124,12 @@ def choose(items: Sequence[Any], rng: Optional[random.Random] = None) -> Any:
 @dataclass
 class EvalResult:
     score: float
-    subscores: List[float]
+    subscores: list[float]
 
 
-def evaluate_program(program: PredictProgram, dataset: Sequence[Tuple[Any, Any]], metric: Callable[[Any, Any], float]) -> EvalResult:
+def evaluate_program(
+    program: PredictProgram, dataset: Sequence[tuple[Any, Any]], metric: Callable[[Any, Any], float]
+) -> EvalResult:
     subs = []
     for x, y in dataset:
         yhat = program.run(x)
@@ -140,8 +144,8 @@ def evaluate_program(program: PredictProgram, dataset: Sequence[Tuple[Any, Any]]
 
 def mipro_v2_compile(
     student: PredictProgram,
-    trainset: Sequence[Tuple[Any, Any]],
-    valset: Sequence[Tuple[Any, Any]],
+    trainset: Sequence[tuple[Any, Any]],
+    valset: Sequence[tuple[Any, Any]],
     metric: Callable[[Any, Any], float],
     *,
     prompt_model: Any,
@@ -159,7 +163,7 @@ def mipro_v2_compile(
     data_aware: bool = True,
     tip_aware: bool = True,
     fewshot_aware: bool = True,
-) -> Tuple[PredictProgram, List[Dict[str, Any]]]:
+) -> tuple[PredictProgram, list[dict[str, Any]]]:
     """MIPROv2-style optimizer.
 
     Arguments mirror the DSPy pseudocode but remain provider-agnostic. The
@@ -171,9 +175,9 @@ def mipro_v2_compile(
     program = student.deepcopy()
 
     # Step 1: bootstrap few-shot example candidates
-    demo_candidates: List[Dict[str, Any]] = []
+    demo_candidates: list[dict[str, Any]] = []
     for _ in range(num_candidates):
-        boot: List[Tuple[Any, Any]] = []
+        boot: list[tuple[Any, Any]] = []
         # collect bootstrapped, self-consistent demos
         while len(boot) < max_bootstrapped_demos:
             x, y = rng.choice(trainset)
@@ -184,9 +188,9 @@ def mipro_v2_compile(
         demo_candidates.append({"boot": boot, "labeled": labeled})
 
     # Step 2: propose instruction candidates per predictor
-    instr_candidates: Dict[str, List[str]] = {}
-    for pred in (program.predictors or ["predictor"]):
-        ctx: Dict[str, Any] = {}
+    instr_candidates: dict[str, list[str]] = {}
+    for pred in program.predictors or ["predictor"]:
+        ctx: dict[str, Any] = {}
         if data_aware:
             ctx["dataset_summary"] = summarize_dataset(trainset)
         if program_aware:
@@ -199,12 +203,12 @@ def mipro_v2_compile(
         instr_candidates[pred] = list(cand)
 
     # Step 3: Bayesian-optimization-like search (random proposer placeholder)
-    history: List[Tuple[Dict[str, Any], float]] = []
-    records: List[Dict[str, Any]] = []
+    history: list[tuple[dict[str, Any], float]] = []
+    records: list[dict[str, Any]] = []
     best_score = -1.0
-    best_cfg: Optional[Dict[str, Any]] = None
+    best_cfg: dict[str, Any] | None = None
 
-    def propose(history_: List[Tuple[Dict[str, Any], float]]) -> Dict[str, Any]:
+    def propose(history_: list[tuple[dict[str, Any], float]]) -> dict[str, Any]:
         # Placeholder: randomly sample from the cartesian product
         instructions = {pred: choose(instr_candidates[pred], rng) for pred in instr_candidates}
         demos = choose(demo_candidates, rng) if demo_candidates else None
@@ -227,15 +231,17 @@ def mipro_v2_compile(
         batch_res = evaluate_program(program_t, batch, metric)
         s_t = batch_res.score
         history.append((theta, s_t))
-        records.append({
-            "trial": t,
-            "evaluation": "batch" if minibatch else "full",
-            "score": s_t,
-            "intervention": {
-                "instructions": theta.get("instructions"),
-                "demo_set": theta.get("demo_set"),
-            },
-        })
+        records.append(
+            {
+                "trial": t,
+                "evaluation": "batch" if minibatch else "full",
+                "score": s_t,
+                "intervention": {
+                    "instructions": theta.get("instructions"),
+                    "demo_set": theta.get("demo_set"),
+                },
+            }
+        )
 
         if (not minibatch) or (t % max(1, minibatch_full_eval_steps) == 0):
             full_res = evaluate_program(program_t, valset, metric)
@@ -243,15 +249,17 @@ def mipro_v2_compile(
             if s_full > best_score:
                 best_score = s_full
                 best_cfg = theta
-            records.append({
-                "trial": t,
-                "evaluation": "full",
-                "score": s_full,
-                "intervention": {
-                    "instructions": theta.get("instructions"),
-                    "demo_set": theta.get("demo_set"),
-                },
-            })
+            records.append(
+                {
+                    "trial": t,
+                    "evaluation": "full",
+                    "score": s_full,
+                    "intervention": {
+                        "instructions": theta.get("instructions"),
+                        "demo_set": theta.get("demo_set"),
+                    },
+                }
+            )
 
     if best_cfg is None:
         return program, records
@@ -275,6 +283,7 @@ __all__ = [
 class ExampleTwoStepDag:
     pass
 
+
 """
 A -> B
 """