synth-ai 0.2.8.dev2__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/learning/prompts/run_random_search_banking77.py

@@ -1,324 +0,0 @@
-"""
-Example: Random Search optimizer on Banking77 using Groq gpt-oss-20b.
-
-Requires:
-- .env with GROQ_API_KEY
-- datasets (`uv add datasets` if needed)
-
-Run:
-- uv run -q python -m synth_ai.learning.prompts.run_random_search_banking77
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-import random
-import time
-from collections.abc import Sequence
-from dataclasses import dataclass, replace
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Any
-
-from datasets import load_dataset
-from dotenv import load_dotenv
-from synth_ai.learning.prompts.random_search import random_search_compile
-from synth_ai.lm.core.main_v3 import LM, build_messages
-from tqdm import tqdm
-
-
-def choose_label(pred: str, label_names: list[str]) -> str:
-    norm = (pred or "").strip().lower()
-    d = {ln.lower(): ln for ln in label_names}
-    if norm in d:
-        return d[norm]
-
-    def score(cand: str) -> int:
-        c = cand.lower()
-        return sum(1 for w in c.split() if w in norm)
-
-    return max(label_names, key=score)
-
-
-def accuracy(pred: str, gold: str, labels: list[str]) -> float:
-    return 1.0 if choose_label(pred, labels) == gold else 0.0
-
-
-@dataclass
-class StudentProgram:
-    lm: LM
-    label_names: list[str]
-    instruction: str
-    demos: list[tuple[str, str]]
-
-    def reset_copy(self):
-        return replace(self, instruction=self.instruction, demos=list(self.demos))
-
-    def deepcopy(self):
-        return replace(self, instruction=str(self.instruction), demos=list(self.demos))
-
-    def with_demos(self, demos: list[tuple[str, str]]):
-        return replace(self, demos=list(demos))
-
-    def run(self, x: str) -> str:
-        # Build a prompt with optional demos
-        examples = "\n".join(f"Input: {a}\nLabel: {b}" for a, b in self.demos)
-        sys = self.instruction or "You are an intent classifier for Banking77."
-        user = (f"Examples:\n{examples}\n\n" if examples else "") + f"Message: {x}\nLabel:"
-        messages = build_messages(sys, user, images_bytes=None, model_name=self.lm.model)
-
-        # Call LM synchronously via asyncio
-        async def _call():
-            resp = await self.lm.respond_async(messages=messages)
-            return (resp.raw_response or "").strip()
-
-        return asyncio.run(_call())
-
-    async def _apredict(self, x: str):
-        examples = "\n".join(f"Input: {a}\nLabel: {b}" for a, b in self.demos)
-        sys = self.instruction or "You are an intent classifier for Banking77."
-        user = (f"Examples:\n{examples}\n\n" if examples else "") + f"Message: {x}\nLabel:"
-        messages = build_messages(sys, user, images_bytes=None, model_name=self.lm.model)
-        resp = await self.lm.respond_async(messages=messages)
-        return (resp.raw_response or "").strip(), (resp.usage or {})
-
-
-def main():
-    load_dotenv()
-    random.seed(0)
-
-    model = os.getenv("MODEL", "openai/gpt-oss-20b")
-    vendor = os.getenv("VENDOR", "groq")
-    lm = LM(model=model, vendor=vendor, temperature=0.0)
-
-    print("Loading Banking77 dataset (train/dev split of test for demo)...")
-    ds = load_dataset("banking77")
-    label_names: list[str] = ds["test"].features["label"].names  # type: ignore
-
-    # Create small train/val from the test split for speed
-    all_items = [(r["text"], label_names[int(r["label"])]) for r in ds["test"]]
-    random.shuffle(all_items)
-    trainset: Sequence[tuple[str, str]] = all_items[:40]
-    valset: Sequence[tuple[str, str]] = all_items[40:60]  # 20 examples
-
-    student = StudentProgram(
-        lm=lm,
-        label_names=label_names,
-        instruction="You are an intent classifier for the Banking77 dataset. Return exactly one label.",
-        demos=[],
-    )
-
-    def metric(yhat: str, y: str) -> float:
-        return accuracy(yhat, y, label_names)
-
-    total_candidates = 3 + 3  # zero-shot, labeled few-shot, bootstrapped + 3 random seeds
-    print(
-        f"Running Random Search optimizer ({total_candidates} candidates, parallel eval of 20 questions)..."
-    )
-
-    def eval_parallel(program: StudentProgram, dataset: Sequence[tuple[str, str]], metric_fn):
-        async def _run():
-            xs = [x for x, _ in dataset]
-            ys = [y for _, y in dataset]
-            preds: list[Optional[str]] = [None] * len(xs)
-            sem = asyncio.Semaphore(int(os.getenv("CONCURRENCY", "5")))
-
-            async def worker(i: int, x: str, y: str):
-                import time
-
-                t_start = time.monotonic()
-                try:
-                    async with sem:
-                        pred, usage = await asyncio.wait_for(
-                            program._apredict(x),
-                            timeout=float(os.getenv("TIMEOUT_S", "45")),
-                        )
-                        t_end = time.monotonic()
-                        return i, y, pred, t_start, t_end, usage or {}
-                except asyncio.CancelledError:
-                    # Respect cancellation but return a placeholder record so scheduler can proceed
-                    t_end = time.monotonic()
-                    return i, y, "", t_start, t_end, {}
-                except Exception:
-                    t_end = time.monotonic()
-                    return i, y, "", t_start, t_end, {}
-
-            tasks = [asyncio.create_task(worker(i, x, y)) for i, (x, y) in enumerate(zip(xs, ys, strict=False))]
-            correct_sum = 0.0
-            processed = 0
-            import statistics
-            import time
-
-            durations: list[float] = []
-            in_tok_sum = 0
-            out_tok_sum = 0
-            in_tok_count = 0
-            out_tok_count = 0
-            details: list[dict[str, Any]] = []
-            t_batch_start = time.monotonic()
-            deadline = float(os.getenv("BATCH_DEADLINE_S", "20"))
-            with tqdm(total=len(tasks), desc="Rollouts", leave=False) as pbar:
-                pending = set(tasks)
-                # Process completions until all done or deadline reached
-                while pending:
-                    elapsed = time.monotonic() - t_batch_start
-                    remaining = max(0.0, deadline - elapsed)
-                    if remaining <= 0.0:
-                        # Cancel any remaining
-                        for t in pending:
-                            t.cancel()
-                        done, _ = await asyncio.wait(pending, return_when=asyncio.ALL_COMPLETED)
-                        # Record canceled as zeros
-                        for task in done:
-                            try:
-                                i, y_true, pred, t_start, t_end, usage = task.result()
-                            except Exception:
-                                # Unknown index: we can't recover; skip as it's canceled before start
-                                continue
-                            # Already processed ones shouldn't be in pending; skip
-                        break
-                    # Wait for at least one completion within remaining time (polling granularity <= 1s)
-                    timeout = min(1.0, remaining)
-                    done, pending = await asyncio.wait(
-                        pending, timeout=timeout, return_when=asyncio.FIRST_COMPLETED
-                    )
-                    import contextlib
-                    for task in done:
-                        try:
-                            i, y_true, pred, t_start, t_end, usage = task.result()
-                        except BaseException:
-                            # Treat as failure/cancelled
-                            continue
-                        durations.append(max(0.0, t_end - t_start))
-                        preds[i] = pred
-                        processed += 1
-                        with contextlib.suppress(Exception):
-                            correct_sum += float(metric_fn(pred, y_true))
-                        with contextlib.suppress(Exception):
-                            pt = usage.get("prompt_tokens") or usage.get("input_tokens")
-                            ct = usage.get("completion_tokens") or usage.get("output_tokens")
-                            if isinstance(pt, (int, float)):
-                                in_tok_sum += int(pt)
-                                in_tok_count += 1
-                            if isinstance(ct, (int, float)):
-                                out_tok_sum += int(ct)
-                                out_tok_count += 1
-                        details.append(
-                            {
-                                "index": i,
-                                "seconds": max(0.0, t_end - t_start),
-                                "score": float(metric_fn(pred, y_true)),
-                                "usage": {
-                                    "prompt_tokens": usage.get("prompt_tokens")
-                                    or usage.get("input_tokens"),
-                                    "completion_tokens": usage.get("completion_tokens")
-                                    or usage.get("output_tokens"),
-                                },
-                            }
-                        )
-                        pbar.update(1)
-                        med = statistics.median(durations) if durations else 0.0
-                        mx = max(durations) if durations else 0.0
-                        avg_in = (in_tok_sum / in_tok_count) if in_tok_count else 0.0
-                        avg_out = (out_tok_sum / out_tok_count) if out_tok_count else 0.0
-                        pbar.set_postfix(
-                            {
-                                "acc": f"{(correct_sum / processed):.2f}",
-                                "done": f"{processed}/{len(tasks)}",
-                                "med_s": f"{med:.1f}",
-                                "max_s": f"{mx:.1f}",
-                                "tin": f"{avg_in:.1f}",
-                                "tout": f"{avg_out:.1f}",
-                            }
-                        )
-            # Compute score only from completed/successful rollouts (drop timeouts/cancelled)
-            subs = [float(d.get("score", 0.0)) for d in details]
-            result = SimpleNamespace(score=(sum(subs) / max(1, len(subs))), subscores=subs)
-            result.details = details
-            result.mean_in = (in_tok_sum / in_tok_count) if in_tok_count else 0.0
-            result.mean_out = (out_tok_sum / out_tok_count) if out_tok_count else 0.0
-            return result
-
-        return asyncio.run(_run())
-
-    pbar = tqdm(total=total_candidates, desc="Candidates")
-    candidate_eval_details: dict[int, Any] = {}
-
-    def on_cand(idx: int, score: float, res, intervention):
-        pbar.update(1)
-        pbar.set_postfix({"score": f"{score:.2f}"})
-        # store per-instance details (for apples-to-apples)
-        import contextlib
-        with contextlib.suppress(Exception):
-            candidate_eval_details[idx] = {
-                "score": score,
-                "mean_in": getattr(res, "mean_in", None),
-                "mean_out": getattr(res, "mean_out", None),
-                "instances": getattr(res, "details", None),
-            }
-        # visible summary line per candidate
-        kind = (
-            intervention.get("kind", "candidate") if isinstance(intervention, dict) else "candidate"
-        )
-        label = intervention.get("label") if isinstance(intervention, dict) else None
-        seed = intervention.get("seed") if isinstance(intervention, dict) else None
-        processed = len(getattr(res, "details", []) or [])
-        from tqdm import tqdm as _tqdm
-
-        _tqdm.write(
-            f"Candidate {idx}/{total_candidates} [{kind}{'' if label is None else f', label={label}'}{'' if seed is None else f', seed={seed}'}]: "
-            f"score={score:.2f} | mean tin/tout={getattr(res, 'mean_in', 0):.1f}/{getattr(res, 'mean_out', 0):.1f} | N={processed}"
-        )
-
-    best, records = random_search_compile(
-        student=student,
-        trainset=trainset,
-        valset=valset,
-        metric=metric,
-        evaluate_fn=eval_parallel,
-        max_bootstrapped_demos=0,
-        max_labeled_demos=4,
-        max_rounds=2,
-        num_candidate_programs=3,
-        on_candidate_evaluated=on_cand,
-    )
-    pbar.close()
-
-    # Evaluate best on holdout (valset) with parallel rollouts
-    print("Evaluating best program on val (parallel rollouts)...")
-    best_res = eval_parallel(best, valset, metric)
-    correct = int(round(best_res.score * max(1, len(best_res.subscores))))
-    print(
-        "Best program accuracy on val: "
-        f"{correct}/{len(valset)} ({best_res.score:.2%}) "
-        f"| mean tokens in/out: {getattr(best_res, 'mean_in', 0):.1f}/{getattr(best_res, 'mean_out', 0):.1f}"
-    )
-
-    # Save per-candidate scores and interventions
-    out = {
-        "context": {
-            "model": model,
-            "vendor": vendor,
-            "train_size": len(trainset),
-            "val_size": len(valset),
-        },
-        "candidates": records,
-        "candidate_eval_details": candidate_eval_details,
-        "best_eval_details": {
-            "score": best_res.score,
-            "mean_in": getattr(best_res, "mean_in", None),
-            "mean_out": getattr(best_res, "mean_out", None),
-            "instances": getattr(best_res, "details", None),
-        },
-    }
-    out_dir = Path(__file__).parent
-    fname = str(out_dir / f"random_search_banking77_{int(time.time())}.json")
-    with open(fname, "w") as f:
-        json.dump(out, f, indent=2)
-    print(f"Saved candidate records to {fname}")
-
-
-if __name__ == "__main__":
-    main()

synth_ai/learning/sse.py

@@ -1,58 +0,0 @@
-from __future__ import annotations
-
-import json
-import time
-from typing import Any, Callable, Optional
-
-import aiohttp
-
-
-def _api_base(b: str) -> str:
-    b = (b or "").rstrip("/")
-    return b if b.endswith("/api") else f"{b}/api"
-
-
-async def stream_events(
-    base_url: str,
-    api_key: str,
-    job_id: str,
-    *,
-    seconds: int = 60,
-    on_event: Optional[Callable[[dict], None]] = None,
-) -> None:
-    if seconds <= 0:
-        return
-    headers = {"Accept": "text/event-stream", "Authorization": f"Bearer {api_key}"}
-    candidates = [
-        f"{_api_base(base_url)}/rl/jobs/{job_id}/events?since_seq=0",
-        f"{_api_base(base_url)}/learning/jobs/{job_id}/events?since_seq=0",
-    ]
-    for url in candidates:
-        try:
-            async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session:
-                async with session.get(url, headers=headers) as resp:
-                    if resp.status != 200:
-                        continue
-                    start_t = time.time()
-                    async for raw in resp.content:
-                        line = raw.decode(errors="ignore").strip()
-                        if not line or line.startswith(":"):
-                            continue
-                        if not line.startswith("data:"):
-                            continue
-                        data = line[5:].strip()
-                        try:
-                            obj = json.loads(data)
-                        except Exception:
-                            continue
-                        if on_event:
-                            try:
-                                on_event(obj)
-                            except Exception:
-                                pass
-                        if (time.time() - start_t) >= seconds:
-                            return
-        except Exception:
-            continue
-
-

synth_ai/learning/validators.py

@@ -1,48 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-import json
-from typing import Any, Dict
-from urllib.parse import urlparse
-
-
-def validate_training_jsonl(path: str | Path, *, sample_lines: int = 50) -> None:
-    p = Path(path)
-    if not p.exists():
-        raise FileNotFoundError(str(p))
-    lines = p.read_text().splitlines()
-    if not lines:
-        raise ValueError("empty JSONL")
-    for i, line in enumerate(lines[: max(1, sample_lines) ], start=1):
-        if not line.strip():
-            continue
-        try:
-            obj = json.loads(line)
-        except Exception as e:
-            raise ValueError(f"invalid json on line {i}: {e}") from e
-        msgs = obj.get("messages")
-        if not isinstance(msgs, list) or len(msgs) < 2:
-            raise ValueError(f"line {i}: missing messages[] with at least 2 turns")
-        roles = [m.get("role") for m in msgs if isinstance(m, dict)]
-        if not roles or not isinstance(roles[0], str):
-            raise ValueError(f"line {i}: missing first role")
-        for m in msgs:
-            if not isinstance(m, dict):
-                raise ValueError(f"line {i}: non-dict message")
-            if not isinstance(m.get("role"), str) or not isinstance(m.get("content"), str) or not m["content"].strip():
-                raise ValueError(f"line {i}: invalid role/content")
-
-
-def validate_task_app_url(url: str, *, name: str = "TASK_APP_BASE_URL") -> None:
-    from synth_ai.task.validators import validate_task_app_url as _vt
-
-    _vt(url, name=name)
-
-
-def validate_trainer_cfg_rl(trainer: Dict[str, Any]) -> None:
-    bs = int(trainer.get("batch_size", 1))
-    gs = int(trainer.get("group_size", 2))
-    if bs < 1:
-        raise ValueError("trainer.batch_size must be >= 1")
-    if gs < 2:
-        raise ValueError("trainer.group_size must be >= 2")

synth_ai/lm/__init__.py

@@ -1,51 +0,0 @@
-"""
-Synth AI Language Model Interface.
-
-Provides a unified interface for multiple LLM providers including OpenAI and Synth.
-"""
-
-from .config import OpenAIConfig, SynthConfig
-from .core.main_v3 import LM
-from .unified_interface import (
-    OpenAIProvider,
-    SynthProvider,
-    UnifiedLMClient,
-    UnifiedLMProvider,
-    create_provider,
-)
-from .vendors.synth_client import (
-    AsyncSynthClient,
-    SyncSynthClient,
-    create_async_client,
-    create_chat_completion_async,
-    create_chat_completion_sync,
-    create_sync_client,
-)
-from .warmup import get_warmup_status, warmup_synth_model
-
-__all__ = [
-    # Configuration
-    "SynthConfig",
-    "OpenAIConfig",
-    # Warmup utilities
-    "warmup_synth_model",
-    "get_warmup_status",
-    # Unified interface
-    "UnifiedLMProvider",
-    "OpenAIProvider",
-    "SynthProvider",
-    "UnifiedLMClient",
-    "create_provider",
-    # Synth client
-    "AsyncSynthClient",
-    "SyncSynthClient",
-    "create_async_client",
-    "create_sync_client",
-    "create_chat_completion_async",
-    "create_chat_completion_sync",
-    # Core LM class
-    "LM",
-]
-
-# Version info
-__version__ = "0.1.0"

synth_ai/lm/caching/constants.py

@@ -1,6 +0,0 @@
-"""
-Constants for the caching module.
-"""
-
-# Maximum size limit for disk cache storage (10GB)
-DISKCACHE_SIZE_LIMIT = 10 * 1024 * 1024 * 1024  # 10GB

synth_ai/lm/caching/ephemeral.py

@@ -1,102 +0,0 @@
-"""
-Ephemeral (temporary) caching for LM responses.
-
-This module provides a disk-based cache that persists only for the duration
-of the application run, useful for avoiding redundant API calls within a session.
-"""
-
-import os
-from dataclasses import dataclass
-
-from diskcache import Cache
-from pydantic import BaseModel
-
-from synth_ai.lm.caching.constants import DISKCACHE_SIZE_LIMIT
-from synth_ai.lm.vendors.base import BaseLMResponse
-
-
-@dataclass
-class EphemeralCache:
-    """
-    Ephemeral cache implementation using diskcache.
-
-    This cache stores LM responses temporarily on disk with a size limit.
-    The cache is cleared when the application restarts.
-    """
-
-    def __init__(self, fast_cache_dir: str = ".cache/ephemeral_cache"):
-        os.makedirs(fast_cache_dir, exist_ok=True)
-        self.fast_cache = Cache(fast_cache_dir, size_limit=DISKCACHE_SIZE_LIMIT)
-
-    def hit_cache(
-        self, key: str, response_model: BaseModel | None = None
-    ) -> BaseLMResponse | None:
-        """
-        Check if a response exists in cache for the given key.
-
-        Args:
-            key: Cache key to look up
-            response_model: Optional Pydantic model to reconstruct structured output
-
-        Returns:
-            BaseLMResponse if found in cache, None otherwise
-        """
-        if key not in self.fast_cache:
-            return None
-
-        try:
-            cache_data = self.fast_cache[key]
-        except AttributeError:
-            return None
-
-        if not isinstance(cache_data, dict):
-            return BaseLMResponse(raw_response=cache_data, structured_output=None, tool_calls=None)
-
-        raw_response = cache_data.get("raw_response")
-        tool_calls = cache_data.get("tool_calls")
-        structured_output = cache_data.get("structured_output")
-
-        if response_model and structured_output:
-            structured_output = response_model(**structured_output)
-
-        return BaseLMResponse(
-            raw_response=raw_response,
-            structured_output=structured_output,
-            tool_calls=tool_calls,
-        )
-
-    def add_to_cache(self, key: str, response: BaseLMResponse | str) -> None:
-        """
-        Add a response to the cache.
-
-        Args:
-            key: Cache key to store under
-            response: Either a BaseLMResponse object or raw string response
-
-        Raises:
-            ValueError: If response type is not supported
-        """
-        if isinstance(response, str):
-            self.fast_cache[key] = response
-            return
-
-        if isinstance(response, BaseLMResponse):
-            cache_data = {
-                "raw_response": response.raw_response
-                if response.raw_response is not None
-                else None,
-                "tool_calls": response.tool_calls if response.tool_calls is not None else None,
-                "structured_output": (
-                    response.structured_output.model_dump()
-                    if response.structured_output is not None
-                    else None
-                ),
-            }
-            self.fast_cache[key] = cache_data
-            return
-
-        raise ValueError(f"Invalid response type: {type(response)}")
-
-    def close(self):
-        """Close the cache and free resources."""
-        self.fast_cache.close()