synth-ai 0.2.6.dev1__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/jobs/client.py

@@ -1,246 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Dict, Optional
-
-from synth_ai.http import AsyncHttpClient
-
-
-class FilesApi:
-    def __init__(self, http: AsyncHttpClient) -> None:
-        self._http = http
-
-    async def upload(self, *, filename: str, content: bytes, purpose: str, content_type: Optional[str] = None, idempotency_key: Optional[str] = None) -> Dict[str, Any]:
-        data = {"purpose": purpose}
-        files = {"file": (filename, content, content_type)}
-        headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
-        return await self._http.post_multipart("/api/files", data=data, files=files, headers=headers)
-
-    async def list(self, *, purpose: Optional[str] = None, after: Optional[str] = None, limit: int = 20) -> Dict[str, Any]:
-        params: Dict[str, Any] = {}
-        if purpose is not None:
-            params["purpose"] = purpose
-        if after is not None:
-            params["after"] = after
-        params["limit"] = limit
-        return await self._http.get("/api/files", params=params)
-
-    async def retrieve(self, file_id: str) -> Dict[str, Any]:
-        return await self._http.get(f"/api/files/{file_id}")
-
-    async def delete(self, file_id: str) -> Any:
-        return await self._http.delete(f"/api/files/{file_id}")
-
-    async def list_jobs(self, file_id: str, *, after: Optional[str] = None, limit: int = 20) -> Dict[str, Any]:
-        params: Dict[str, Any] = {"limit": limit}
-        if after is not None:
-            params["after"] = after
-        return await self._http.get(f"/api/files/{file_id}/jobs", params=params)
-
-
-class SftJobsApi:
-    def __init__(self, http: AsyncHttpClient) -> None:
-        self._http = http
-
-    async def create(
-        self,
-        *,
-        training_file: str,
-        model: str,
-        validation_file: Optional[str] = None,
-        hyperparameters: Optional[Dict[str, Any]] = None,
-        suffix: Optional[str] = None,
-        integrations: Optional[Dict[str, Any]] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-        idempotency_key: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        payload: Dict[str, Any] = {
-            "training_file": training_file,
-            "model": model,
-        }
-        if validation_file is not None:
-            payload["validation_file"] = validation_file
-        if hyperparameters is not None:
-            payload["hyperparameters"] = hyperparameters
-        if suffix is not None:
-            payload["suffix"] = suffix
-        if integrations is not None:
-            payload["integrations"] = integrations
-        if metadata is not None:
-            payload["metadata"] = metadata
-        headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
-        return await self._http.post_json("/api/sft/jobs", json=payload, headers=headers)
-
-    async def list(
-        self,
-        *,
-        status: Optional[str] = None,
-        model: Optional[str] = None,
-        file_id: Optional[str] = None,
-        created_after: Optional[int] = None,
-        created_before: Optional[int] = None,
-        after: Optional[str] = None,
-        limit: int = 20,
-    ) -> Dict[str, Any]:
-        params: Dict[str, Any] = {"limit": limit}
-        if status is not None:
-            params["status"] = status
-        if model is not None:
-            params["model"] = model
-        if file_id is not None:
-            params["file_id"] = file_id
-        if created_after is not None:
-            params["created_after"] = created_after
-        if created_before is not None:
-            params["created_before"] = created_before
-        if after is not None:
-            params["after"] = after
-        return await self._http.get("/api/sft/jobs", params=params)
-
-    async def retrieve(self, job_id: str) -> Dict[str, Any]:
-        return await self._http.get(f"/api/sft/jobs/{job_id}")
-
-    async def cancel(self, job_id: str) -> Dict[str, Any]:
-        return await self._http.post_json(f"/api/sft/jobs/{job_id}/cancel", json={})
-
-    async def list_events(self, job_id: str, *, since_seq: int = 0, limit: int = 200) -> Dict[str, Any]:
-        params = {"since_seq": since_seq, "limit": limit}
-        return await self._http.get(f"/api/sft/jobs/{job_id}/events", params=params)
-
-    async def checkpoints(self, job_id: str, *, after: Optional[str] = None, limit: int = 10) -> Dict[str, Any]:
-        params: Dict[str, Any] = {"limit": limit}
-        if after is not None:
-            params["after"] = after
-        return await self._http.get(f"/api/sft/jobs/{job_id}/checkpoints", params=params)
-
-
-class RlJobsApi:
-    def __init__(self, http: AsyncHttpClient) -> None:
-        self._http = http
-
-    async def create(
-        self,
-        *,
-        model: str,
-        endpoint_base_url: str,
-        trainer_id: str,
-        trainer: Optional[Dict[str, Any]] = None,
-        job_config_id: Optional[str] = None,
-        config: Optional[Dict[str, Any]] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-        idempotency_key: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        payload: Dict[str, Any] = {
-            "model": model,
-            "endpoint_base_url": endpoint_base_url,
-            "trainer_id": trainer_id,
-        }
-        if trainer is not None:
-            payload["trainer"] = trainer
-        if job_config_id is not None:
-            payload["job_config_id"] = job_config_id
-        if config is not None:
-            payload["config"] = config
-        if metadata is not None:
-            payload["metadata"] = metadata
-        headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
-        return await self._http.post_json("/api/rl/jobs", json=payload, headers=headers)
-
-    async def list(
-        self,
-        *,
-        status: Optional[str] = None,
-        model: Optional[str] = None,
-        created_after: Optional[int] = None,
-        created_before: Optional[int] = None,
-        after: Optional[str] = None,
-        limit: int = 20,
-    ) -> Dict[str, Any]:
-        params: Dict[str, Any] = {"limit": limit}
-        if status is not None:
-            params["status"] = status
-        if model is not None:
-            params["model"] = model
-        if created_after is not None:
-            params["created_after"] = created_after
-        if created_before is not None:
-            params["created_before"] = created_before
-        if after is not None:
-            params["after"] = after
-        return await self._http.get("/api/rl/jobs", params=params)
-
-    async def retrieve(self, job_id: str) -> Dict[str, Any]:
-        return await self._http.get(f"/api/rl/jobs/{job_id}")
-
-    async def cancel(self, job_id: str) -> Dict[str, Any]:
-        return await self._http.post_json(f"/api/rl/jobs/{job_id}/cancel", json={})
-
-    async def list_events(self, job_id: str, *, since_seq: int = 0, limit: int = 200) -> Dict[str, Any]:
-        params = {"since_seq": since_seq, "limit": limit}
-        return await self._http.get(f"/api/rl/jobs/{job_id}/events", params=params)
-
-    async def metrics(self, job_id: str, *, after_step: int = -1, limit: int = 200) -> Dict[str, Any]:
-        params = {"after_step": after_step, "limit": limit}
-        return await self._http.get(f"/api/rl/jobs/{job_id}/metrics", params=params)
-
-
-class ModelsApi:
-    def __init__(self, http: AsyncHttpClient) -> None:
-        self._http = http
-
-    async def list(
-        self,
-        *,
-        source: Optional[str] = None,
-        base_model: Optional[str] = None,
-        status: Optional[str] = None,
-        after: Optional[str] = None,
-        limit: int = 20,
-    ) -> Dict[str, Any]:
-        params: Dict[str, Any] = {"limit": limit}
-        if source is not None:
-            params["source"] = source
-        if base_model is not None:
-            params["base_model"] = base_model
-        if status is not None:
-            params["status"] = status
-        if after is not None:
-            params["after"] = after
-        return await self._http.get("/api/models", params=params)
-
-    async def retrieve(self, model_id: str) -> Dict[str, Any]:
-        return await self._http.get(f"/api/models/{model_id}")
-
-    async def delete(self, model_id: str) -> Any:
-        return await self._http.delete(f"/api/models/{model_id}")
-
-    async def list_jobs(self, model_id: str, *, after: Optional[str] = None, limit: int = 20) -> Dict[str, Any]:
-        params: Dict[str, Any] = {"limit": limit}
-        if after is not None:
-            params["after"] = after
-        return await self._http.get(f"/api/models/{model_id}/jobs", params=params)
-
-
-class JobsClient:
-    """High-level client aggregating job APIs.
-
-    Usage:
-        async with JobsClient(base_url, api_key) as c:
-            await c.files.list()
-    """
-
-    def __init__(self, base_url: str, api_key: str, timeout: float = 30.0, http: Optional[AsyncHttpClient] = None) -> None:
-        self._base_url = base_url
-        self._api_key = api_key
-        self._timeout = timeout
-        self._http = http or AsyncHttpClient(base_url, api_key, timeout=timeout)
-        self.files = FilesApi(self._http)
-        self.sft = SftJobsApi(self._http)
-        self.rl = RlJobsApi(self._http)
-        self.models = ModelsApi(self._http)
-
-    async def __aenter__(self) -> "JobsClient":
-        await self._http.__aenter__()
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb) -> None:  # noqa: ANN001
-        await self._http.__aexit__(exc_type, exc, tb)

synth_ai/learning/__init__.py

@@ -1,24 +0,0 @@
-from .client import LearningClient
-from .rl_client import RlClient
-from .ft_client import FtClient
-from .validators import validate_training_jsonl, validate_trainer_cfg_rl
-from synth_ai.task import validate_task_app_url, task_app_health
-from .health import backend_health, pricing_preflight, balance_autumn_normalized
-from .sse import stream_events as stream_job_events
-from .jobs import JobHandle, JobsApiResolver
-
-__all__ = [
-    "LearningClient",
-    "RlClient",
-    "FtClient",
-    "validate_training_jsonl",
-    "validate_trainer_cfg_rl",
-    "validate_task_app_url",
-    "backend_health",
-    "task_app_health",
-    "pricing_preflight",
-    "balance_autumn_normalized",
-    "stream_job_events",
-    "JobHandle",
-    "JobsApiResolver",
-]

synth_ai/learning/config.py

@@ -1,43 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Any, Dict, Optional
-
-
-@dataclass
-class FTJobConfig:
-    model: str
-    training_file_id: str
-    n_epochs: int = 1
-    batch_size: int = 1
-    upload_to_wasabi: bool = True
-
-    def hyperparameters(self) -> Dict[str, Any]:
-        if self.n_epochs < 1:
-            raise ValueError("n_epochs must be >= 1")
-        if self.batch_size < 1:
-            raise ValueError("batch_size must be >= 1")
-        return {"n_epochs": int(self.n_epochs), "batch_size": int(self.batch_size)}
-
-    def metadata(self) -> Dict[str, Any]:  # type: ignore[override]
-        return {"upload_to_wasabi": bool(self.upload_to_wasabi)}
-
-
-@dataclass
-class RLJobConfig:
-    model: str
-    task_app_url: str
-    trainer_id: str
-    batch_size: int = 1
-    group_size: int = 2
-    job_config_id: Optional[str] = None
-    inline_config: Optional[Dict[str, Any]] = None
-
-    def trainer_dict(self) -> Dict[str, Any]:
-        if self.batch_size < 1:
-            raise ValueError("batch_size must be >= 1")
-        if self.group_size < 2:
-            raise ValueError("group_size must be >= 2")
-        return {"batch_size": int(self.batch_size), "group_size": int(self.group_size)}
-
-

synth_ai/learning/ft_client.py

@@ -1,59 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Any, Dict, Optional
-
-from ..http import AsyncHttpClient, HTTPError
-
-
-class FtClient:
-    def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0) -> None:
-        self._base_url = base_url.rstrip("/")
-        self._api_key = api_key
-        self._timeout = timeout
-
-    async def upload_training_file(self, path: str | Path, *, purpose: str = "fine-tune") -> str:
-        p = Path(path)
-        content = p.read_bytes()
-        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
-            data = {"purpose": purpose}
-            files = {"file": (p.name, content, _infer_content_type(p.name))}
-            js = await http.post_multipart("/api/learning/files", data=data, files=files)
-        if not isinstance(js, dict) or "id" not in js:
-            raise HTTPError(status=500, url="/api/learning/files", message="invalid_upload_response", body_snippet=str(js)[:200])
-        return str(js["id"])
-
-    async def create_sft_job(
-        self,
-        *,
-        model: str,
-        training_file_id: str,
-        hyperparameters: Dict[str, Any],
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> Dict[str, Any]:
-        body = {
-            "training_type": "sft_offline",
-            "model": model,
-            "training_file_id": training_file_id,
-            "hyperparameters": dict(hyperparameters or {}),
-            "metadata": dict(metadata or {}),
-        }
-        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
-            return await http.post_json("/api/learning/jobs", json=body)
-
-    async def start_job(self, job_id: str) -> Dict[str, Any]:
-        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
-            return await http.post_json(f"/api/learning/jobs/{job_id}/start", json={})
-
-
-def _infer_content_type(filename: str) -> str:
-    name = filename.lower()
-    if name.endswith(".jsonl"):
-        return "application/jsonl"
-    if name.endswith(".json"):
-        return "application/json"
-    if name.endswith(".txt"):
-        return "text/plain"
-    return "application/octet-stream"
-
-

synth_ai/learning/offline/providers.py

@@ -1,7 +0,0 @@
-## Gemini
-
-
-## OpenAI
-
-
-## Synth

synth_ai/learning/prompts/banking77_injection_eval.py

@@ -1,168 +0,0 @@
-"""
-Banking77 in-context injection evals (async, not tests)
-
-Samples a handful of Banking77 prompts and evaluates multiple override
-contexts in parallel, printing simple accuracy for each.
-
-Usage
-- Keys in .env (GROQ_API_KEY, etc.)
-- Run: uv run -q python -m synth_ai.learning.prompts.banking77_injection_eval
-  Optional env:
-    - N_SAMPLES=20 (default)
-    - MODEL=openai/gpt-oss-20b (default)
-    - VENDOR=groq (default)
-"""
-
-from __future__ import annotations
-
-import asyncio
-import os
-import random
-from typing import Any
-
-from datasets import load_dataset
-from dotenv import load_dotenv
-from synth_ai.lm.core.main_v3 import LM, build_messages
-from synth_ai.lm.overrides import LMOverridesContext
-
-
-async def classify_one(lm: LM, text: str, label_names: list[str]) -> str:
-    labels_joined = ", ".join(label_names)
-    system_message = (
-        "You are an intent classifier for the Banking77 dataset. "
-        "Given a customer message, respond with exactly one label from the list. "
-        "Return only the label text with no extra words.\n\n"
-        f"Valid labels: {labels_joined}"
-    )
-    user_message = f"Message: {text}\nLabel:"
-    messages = build_messages(system_message, user_message, images_bytes=None, model_name=lm.model)
-    resp = await lm.respond_async(messages=messages)
-    return (resp.raw_response or "").strip()
-
-
-def choose_label(pred: str, label_names: list[str]) -> str:
-    norm_pred = pred.strip().lower()
-    label_lookup = {ln.lower(): ln for ln in label_names}
-    mapped = label_lookup.get(norm_pred)
-    if mapped is not None:
-        return mapped
-
-    # Fallback: choose the label with the highest naive token overlap
-    def score(cand: str) -> int:
-        c = cand.lower()
-        return sum(1 for w in c.split() if w in norm_pred)
-
-    return max(label_names, key=score)
-
-
-async def eval_context(
-    lm: LM,
-    items: list[tuple[str, str]],
-    label_names: list[str],
-    ctx_name: str,
-    specs: list[dict[str, Any]],
-) -> tuple[str, int, int]:
-    correct = 0
-    with LMOverridesContext(specs):
-        tasks = [classify_one(lm, text, label_names) for text, _ in items]
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-    for (text, gold), pred in zip(items, results, strict=False):
-        if isinstance(pred, Exception):
-            # Treat exceptions as incorrect
-            continue
-        mapped = choose_label(pred, label_names)
-        correct += int(mapped == gold)
-    return (ctx_name, correct, len(items))
-
-
-async def main() -> None:
-    load_dotenv()
-
-    n = int(os.getenv("N_SAMPLES", "20"))
-    model = os.getenv("MODEL", "openai/gpt-oss-20b")
-    vendor = os.getenv("VENDOR", "groq")
-
-    lm = LM(model=model, vendor=vendor, temperature=0.0)
-
-    print("Loading Banking77 dataset (split='test')...")
-    ds = load_dataset("banking77", split="test")
-    label_names: list[str] = ds.features["label"].names  # type: ignore
-
-    idxs = random.sample(range(len(ds)), k=min(n, len(ds)))
-    items = [
-        (ds[i]["text"], label_names[int(ds[i]["label"])])  # (text, gold_label)
-        for i in idxs
-    ]
-
-    # Define a few override contexts to compare
-    contexts: list[dict[str, Any]] = [
-        {
-            "name": "baseline (no overrides)",
-            "overrides": [],
-        },
-        {
-            "name": "nonsense prompt injection (expected worse)",
-            "overrides": [
-                {
-                    "match": {"contains": "", "role": "user"},
-                    "injection_rules": [
-                        # Heavily corrupt user text by replacing vowels
-                        {"find": "a", "replace": "x"},
-                        {"find": "e", "replace": "x"},
-                        {"find": "i", "replace": "x"},
-                        {"find": "o", "replace": "x"},
-                        {"find": "u", "replace": "x"},
-                        {"find": "A", "replace": "X"},
-                        {"find": "E", "replace": "X"},
-                        {"find": "I", "replace": "X"},
-                        {"find": "O", "replace": "X"},
-                        {"find": "U", "replace": "X"},
-                    ],
-                }
-            ],
-        },
-        {
-            "name": "injection: atm->ATM, txn->transaction",
-            "overrides": [
-                {
-                    "match": {"contains": "atm", "role": "user"},
-                    "injection_rules": [
-                        {"find": "atm", "replace": "ATM"},
-                        {"find": "txn", "replace": "transaction"},
-                    ],
-                }
-            ],
-        },
-        {
-            "name": "params: temperature=0.0",
-            "overrides": [
-                {"match": {"contains": ""}, "params": {"temperature": 0.0}},
-            ],
-        },
-        {
-            "name": "model override: 20b->120b",
-            "overrides": [
-                {"match": {"contains": ""}, "params": {"model": "openai/gpt-oss-120b"}},
-            ],
-        },
-    ]
-
-    print(f"\nEvaluating {len(contexts)} contexts on {len(items)} Banking77 samples (async)...")
-
-    # Evaluate each context sequentially but batched (each context classifies in parallel)
-    results: list[tuple[str, int, int]] = []
-    for ctx in contexts:
-        name = ctx["name"]
-        specs = ctx["overrides"]
-        print(f"Evaluating: {name} ...")
-        res = await eval_context(lm, items, label_names, name, specs)
-        results.append(res)
-
-    print("\nResults:")
-    for name, correct, total in results:
-        acc = correct / total if total else 0.0
-        print(f"- {name}: {correct}/{total} correct ({acc:.2%})")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())