synth-ai 0.2.4.dev6__py3-none-any.whl → 0.2.4.dev8__py3-none-any.whl

synth_ai/learning/config.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class FTJobConfig:
+    model: str
+    training_file_id: str
+    n_epochs: int = 1
+    batch_size: int = 1
+    upload_to_wasabi: bool = True
+
+    def hyperparameters(self) -> Dict[str, Any]:
+        if self.n_epochs < 1:
+            raise ValueError("n_epochs must be >= 1")
+        if self.batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        return {"n_epochs": int(self.n_epochs), "batch_size": int(self.batch_size)}
+
+    def metadata(self) -> Dict[str, Any]:  # type: ignore[override]
+        return {"upload_to_wasabi": bool(self.upload_to_wasabi)}
+
+
+@dataclass
+class RLJobConfig:
+    model: str
+    task_app_url: str
+    trainer_id: str
+    batch_size: int = 1
+    group_size: int = 2
+    job_config_id: Optional[str] = None
+    inline_config: Optional[Dict[str, Any]] = None
+
+    def trainer_dict(self) -> Dict[str, Any]:
+        if self.batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        if self.group_size < 2:
+            raise ValueError("group_size must be >= 2")
+        return {"batch_size": int(self.batch_size), "group_size": int(self.group_size)}
+
+

synth_ai/learning/constants.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+# Terminal statuses normalized across FT and RL
+TERMINAL_STATUSES = {
+    "succeeded",
+    "failed",
+    "cancelled",
+    "canceled",
+    "error",
+    "completed",
+}
+
+# Terminal event types (success/failure) across FT and RL
+TERMINAL_EVENT_SUCCESS = {
+    "sft.completed",
+    "sft.workflow.completed",
+    "rl.job.completed",
+    "rl.train.completed",
+    "workflow.completed",
+}
+
+TERMINAL_EVENT_FAILURE = {
+    "sft.failed",
+    "sft.workflow.failed",
+    "rl.job.failed",
+    "workflow.failed",
+}
+
+

synth_ai/learning/ft_client.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from ..http import AsyncHttpClient, HTTPError
+
+
+class FtClient:
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0) -> None:
+        self._base_url = base_url.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+
+    async def upload_training_file(self, path: str | Path, *, purpose: str = "fine-tune") -> str:
+        p = Path(path)
+        content = p.read_bytes()
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            data = {"purpose": purpose}
+            files = {"file": (p.name, content, _infer_content_type(p.name))}
+            js = await http.post_multipart("/api/learning/files", data=data, files=files)
+        if not isinstance(js, dict) or "id" not in js:
+            raise HTTPError(status=500, url="/api/learning/files", message="invalid_upload_response", body_snippet=str(js)[:200])
+        return str(js["id"])
+
+    async def create_sft_job(
+        self,
+        *,
+        model: str,
+        training_file_id: str,
+        hyperparameters: Dict[str, Any],
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        body = {
+            "training_type": "sft_offline",
+            "model": model,
+            "training_file_id": training_file_id,
+            "hyperparameters": dict(hyperparameters or {}),
+            "metadata": dict(metadata or {}),
+        }
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            return await http.post_json("/api/learning/jobs", json=body)
+
+    async def start_job(self, job_id: str) -> Dict[str, Any]:
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            return await http.post_json(f"/api/learning/jobs/{job_id}/start", json={})
+
+
+def _infer_content_type(filename: str) -> str:
+    name = filename.lower()
+    if name.endswith(".jsonl"):
+        return "application/jsonl"
+    if name.endswith(".json"):
+        return "application/json"
+    if name.endswith(".txt"):
+        return "text/plain"
+    return "application/octet-stream"
+
+

synth_ai/learning/gateway.py

@@ -1,4 +1,2 @@
-
-
 class OfflineGateway:
-    pass
+    pass

synth_ai/learning/health.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+import aiohttp
+
+from ..http import AsyncHttpClient
+
+
+def _api_base(b: str) -> str:
+    b = (b or "").rstrip("/")
+    return b if b.endswith("/api") else f"{b}/api"
+
+
+async def backend_health(base_url: str, api_key: str) -> Dict[str, Any]:
+    async with AsyncHttpClient(base_url, api_key, timeout=15.0) as http:
+        js = await http.get(f"{_api_base(base_url)}/health")
+    return {"ok": True, "raw": js}
+
+
+async def task_app_health(task_app_url: str) -> Dict[str, Any]:
+    # Delegate to central task module for consistency
+    from synth_ai.task.health import task_app_health as _th
+
+    return await _th(task_app_url)
+
+
+async def pricing_preflight(base_url: str, api_key: str, *, job_type: str, gpu_type: str, estimated_seconds: float, container_count: int) -> Dict[str, Any]:
+    body = {
+        "job_type": job_type,
+        "gpu_type": gpu_type,
+        "estimated_seconds": float(estimated_seconds or 0.0),
+        "container_count": int(container_count or 1),
+    }
+    async with AsyncHttpClient(base_url, api_key, timeout=30.0) as http:
+        js = await http.post_json(f"{_api_base(base_url)}/v1/pricing/preflight", json=body)
+    return js if isinstance(js, dict) else {"raw": js}
+
+
+async def balance_autumn_normalized(base_url: str, api_key: str) -> Dict[str, Any]:
+    async with AsyncHttpClient(base_url, api_key, timeout=30.0) as http:
+        js = await http.get(f"{_api_base(base_url)}/v1/balance/autumn-normalized")
+    return js if isinstance(js, dict) else {"raw": js}
+

synth_ai/learning/jobs.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional
+import time
+
+from .constants import TERMINAL_EVENT_FAILURE, TERMINAL_EVENT_SUCCESS, TERMINAL_STATUSES
+from ..http import AsyncHttpClient, sleep
+
+
+def _api_base(b: str) -> str:
+    b = (b or "").rstrip("/")
+    return b if b.endswith("/api") else f"{b}/api"
+
+
+class JobsApiResolver:
+    def __init__(self, base_url: str, *, strict: bool) -> None:
+        self._base = _api_base(base_url)
+        self._strict = strict
+
+    def status_urls(self, job_id: str) -> List[str]:
+        if self._strict:
+            return [f"{self._base}/learning/jobs/{job_id}"]
+        return [
+            f"{self._base}/learning/jobs/{job_id}",
+            f"{self._base}/rl/jobs/{job_id}",
+            f"{self._base}/orchestration/jobs/{job_id}",
+        ]
+
+    def events_urls(self, job_id: str, since: int) -> List[str]:
+        if self._strict:
+            return [f"{self._base}/learning/jobs/{job_id}/events?since_seq={since}&limit=200"]
+        return [
+            f"{self._base}/learning/jobs/{job_id}/events?since_seq={since}&limit=200",
+            f"{self._base}/orchestration/jobs/{job_id}/events?since_seq={since}&limit=200",
+            # RL /jobs/{id}/events is SSE in backend; avoid in JSON poller
+        ]
+
+    def metrics_url(self, job_id: str, after_step: int) -> str:
+        return f"{self._base}/learning/jobs/{job_id}/metrics?after_step={after_step}&limit=200"
+
+
+class JobHandle:
+    def __init__(self, base_url: str, api_key: str, job_id: str, *, strict: bool = True, timeout: float = 600.0) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.job_id = job_id
+        self.strict = strict
+        self.timeout = timeout
+
+    async def poll_until_terminal(
+        self,
+        *,
+        interval_seconds: float = 2.0,
+        max_seconds: float | None = None,
+        empty_polls_threshold: int = 5,
+        startup_deadline_s: int = 45,
+        on_event: Optional[Callable[[Dict[str, Any]], None]] = None,
+        on_metric: Optional[Callable[[Dict[str, Any]], None]] = None,
+    ) -> Dict[str, Any]:
+        last_seq_by_stream: Dict[str, int] = {}
+        events_job_id: Optional[str] = None
+        last_status: Optional[str] = None
+        last_step_by_name: Dict[str, int] = {}
+        empty_polls = 0
+        saw_any_event = False
+        start_t = time.time()
+        resolver = JobsApiResolver(self.base_url, strict=self.strict)
+        detected_fine_tuned_model: Optional[str] = None
+
+        async with AsyncHttpClient(self.base_url, self.api_key, timeout=self.timeout) as http:
+            while True:
+                # Status
+                status_data: Optional[Dict[str, Any]] = None
+                for su in resolver.status_urls(self.job_id):
+                    try:
+                        status_data = await http.get(su)
+                        if isinstance(status_data, dict):
+                            break
+                    except Exception:
+                        continue
+                status = str((status_data or {}).get("status") or "").lower()
+                if status_data:
+                    linked = status_data.get("linked_job_id")
+                    if isinstance(linked, str) and linked and linked != events_job_id:
+                        events_job_id = linked
+                    # Capture fine_tuned_model if already present on status
+                    if not detected_fine_tuned_model:
+                        ftm = status_data.get("fine_tuned_model")
+                        if isinstance(ftm, str) and ftm:
+                            detected_fine_tuned_model = ftm
+                if status and status != last_status:
+                    last_status = status
+                    if on_event:
+                        try:
+                            on_event({"type": "job.status", "message": status})
+                        except Exception:
+                            pass
+
+                # Events
+                stream_ids = [self.job_id]
+                if events_job_id and events_job_id not in stream_ids:
+                    stream_ids.append(events_job_id)
+                total_events_this_cycle = 0
+                terminal_event_seen = False
+                terminal_event_status: Optional[str] = None
+                for ev_id in stream_ids:
+                    since = last_seq_by_stream.get(ev_id, 0)
+                    for eu in resolver.events_urls(ev_id, since):
+                        try:
+                            ev_js = await http.get(eu)
+                        except Exception:
+                            continue
+                        try:
+                            events = (ev_js or {}).get("events") or (ev_js or {}).get("data") or []
+                            if not isinstance(events, list):
+                                events = []
+                        except Exception:
+                            events = []
+                        total_events_this_cycle += len(events)
+                        if events:
+                            saw_any_event = True
+                        for e in events:
+                            seq_val = int(e.get("seq") or 0)
+                            if seq_val <= last_seq_by_stream.get(ev_id, 0):
+                                continue
+                            last_seq_by_stream[ev_id] = seq_val
+                            if on_event:
+                                try:
+                                    on_event(e)
+                                except Exception:
+                                    pass
+                            et = str(e.get("type") or e.get("event_type") or "").lower()
+                            # Capture fine_tuned_model from event data when available
+                            if not detected_fine_tuned_model:
+                                try:
+                                    data_obj = e.get("data") or {}
+                                    ftm = data_obj.get("fine_tuned_model") if isinstance(data_obj, dict) else None
+                                    if isinstance(ftm, str) and ftm:
+                                        detected_fine_tuned_model = ftm
+                                except Exception:
+                                    pass
+                            if et in TERMINAL_EVENT_SUCCESS:
+                                terminal_event_seen = True
+                                terminal_event_status = "succeeded"
+                            elif et in TERMINAL_EVENT_FAILURE:
+                                terminal_event_seen = True
+                                terminal_event_status = "failed"
+
+                # Metrics
+                try:
+                    after = max(last_step_by_name.values()) if last_step_by_name else -1
+                    mu = resolver.metrics_url(self.job_id, after)
+                    md = await http.get(mu)
+                    for p in (md or {}).get("points", []):
+                        name = str(p.get("name") or "")
+                        step = int(p.get("step") or -1)
+                        if step <= last_step_by_name.get(name, -1):
+                            continue
+                        last_step_by_name[name] = step
+                        if on_metric:
+                            try:
+                                on_metric(p)
+                            except Exception:
+                                pass
+                except Exception:
+                    pass
+
+                # Terminal decisions
+                if terminal_event_seen or (status and status in TERMINAL_STATUSES):
+                    # Best-effort enrichment of final result with fine_tuned_model
+                    result_status = terminal_event_status or status or "completed"
+                    final_res: Dict[str, Any] = {"status": result_status, "job_id": self.job_id}
+                    if not detected_fine_tuned_model:
+                        # Briefly try to re-fetch status to see if fine_tuned_model is persisted
+                        try:
+                            for su in resolver.status_urls(self.job_id):
+                                try:
+                                    final_status = await http.get(su)
+                                    if isinstance(final_status, dict):
+                                        ftm2 = final_status.get("fine_tuned_model")
+                                        if isinstance(ftm2, str) and ftm2:
+                                            detected_fine_tuned_model = ftm2
+                                            break
+                                except Exception:
+                                    continue
+                        except Exception:
+                            pass
+                    if detected_fine_tuned_model:
+                        final_res["fine_tuned_model"] = detected_fine_tuned_model
+                    return final_res
+
+                # Guards (relaxed): do not abort on consecutive empty polls
+                if total_events_this_cycle == 0:
+                    empty_polls += 1
+                else:
+                    empty_polls = 0
+                if not saw_any_event and (time.time() - start_t) > int(startup_deadline_s):
+                    raise AssertionError(
+                        f"No events observed within startup window ({startup_deadline_s}s). Investigate event streaming."
+                    )
+                await sleep(interval_seconds)
+                if max_seconds is not None and (time.time() - start_t) >= max_seconds:
+                    raise TimeoutError(f"Polling timed out after {max_seconds}s for job {self.job_id}")
+
+

synth_ai/learning/prompts/banking77_injection_eval.py

@@ -18,16 +18,15 @@ from __future__ import annotations
 import asyncio
 import os
 import random
-from typing import List, Dict, Any, Tuple
+from typing import Any
 
-from dotenv import load_dotenv
 from datasets import load_dataset
-
+from dotenv import load_dotenv
 from synth_ai.lm.core.main_v3 import LM, build_messages
 from synth_ai.lm.overrides import LMOverridesContext
 
 
-async def classify_one(lm: LM, text: str, label_names: List[str]) -> str:
+async def classify_one(lm: LM, text: str, label_names: list[str]) -> str:
     labels_joined = ", ".join(label_names)
     system_message = (
         "You are an intent classifier for the Banking77 dataset. "
@@ -41,7 +40,7 @@ async def classify_one(lm: LM, text: str, label_names: List[str]) -> str:
     return (resp.raw_response or "").strip()
 
 
-def choose_label(pred: str, label_names: List[str]) -> str:
+def choose_label(pred: str, label_names: list[str]) -> str:
     norm_pred = pred.strip().lower()
     label_lookup = {ln.lower(): ln for ln in label_names}
     mapped = label_lookup.get(norm_pred)
@@ -56,12 +55,18 @@ def choose_label(pred: str, label_names: List[str]) -> str:
     return max(label_names, key=score)
 
 
-async def eval_context(lm: LM, items: List[Tuple[str, str]], label_names: List[str], ctx_name: str, specs: List[Dict[str, Any]]) -> Tuple[str, int, int]:
+async def eval_context(
+    lm: LM,
+    items: list[tuple[str, str]],
+    label_names: list[str],
+    ctx_name: str,
+    specs: list[dict[str, Any]],
+) -> tuple[str, int, int]:
     correct = 0
     with LMOverridesContext(specs):
         tasks = [classify_one(lm, text, label_names) for text, _ in items]
         results = await asyncio.gather(*tasks, return_exceptions=True)
-    for (text, gold), pred in zip(items, results):
+    for (text, gold), pred in zip(items, results, strict=False):
         if isinstance(pred, Exception):
             # Treat exceptions as incorrect
             continue
@@ -81,7 +86,7 @@ async def main() -> None:
 
     print("Loading Banking77 dataset (split='test')...")
     ds = load_dataset("banking77", split="test")
-    label_names: List[str] = ds.features["label"].names  # type: ignore
+    label_names: list[str] = ds.features["label"].names  # type: ignore
 
     idxs = random.sample(range(len(ds)), k=min(n, len(ds)))
     items = [
@@ -90,7 +95,7 @@ async def main() -> None:
     ]
 
     # Define a few override contexts to compare
-    contexts: List[Dict[str, Any]] = [
+    contexts: list[dict[str, Any]] = [
         {
             "name": "baseline (no overrides)",
             "overrides": [],
@@ -145,7 +150,7 @@ async def main() -> None:
     print(f"\nEvaluating {len(contexts)} contexts on {len(items)} Banking77 samples (async)...")
 
     # Evaluate each context sequentially but batched (each context classifies in parallel)
-    results: List[Tuple[str, int, int]] = []
+    results: list[tuple[str, int, int]] = []
     for ctx in contexts:
         name = ctx["name"]
         specs = ctx["overrides"]

synth_ai/learning/prompts/hello_world_in_context_injection_ex.py

@@ -27,18 +27,17 @@ from __future__ import annotations
 import asyncio
 import os
 import random
-from typing import Any, Dict, List, Optional
 
 from datasets import load_dataset
 
 # Use the v3 LM class present in this repo
 from synth_ai.lm.core.main_v3 import LM, build_messages
-from synth_ai.tracing_v3.session_tracer import SessionTracer
-from synth_ai.tracing_v3.abstractions import LMCAISEvent
-
 
 # Use Overrides context to demonstrate matching by content
 from synth_ai.lm.overrides import LMOverridesContext
+from synth_ai.tracing_v3.abstractions import LMCAISEvent
+from synth_ai.tracing_v3.session_tracer import SessionTracer
+
 INJECTION_RULES = [
     {"find": "accnt", "replace": "account"},
     {"find": "atm", "replace": "ATM"},
@@ -46,7 +45,7 @@ INJECTION_RULES = [
 ]
 
 
-async def classify_sample(lm: LM, text: str, label_names: List[str]) -> str:
+async def classify_sample(lm: LM, text: str, label_names: list[str]) -> str:
     """Classify one Banking77 utterance and return the predicted label name."""
     labels_joined = ", ".join(label_names)
     system_message = (
@@ -77,7 +76,7 @@ async def main() -> None:
     # Columns: {"text": str, "label": int}; label names at ds.features["label"].names
     print("Loading Banking77 dataset (split='test')...")
     ds = load_dataset("banking77", split="test")
-    label_names: List[str] = ds.features["label"].names  # type: ignore
+    label_names: list[str] = ds.features["label"].names  # type: ignore
 
     # Sample a few items for a quick demo
     n = int(os.getenv("N_SAMPLES", "8"))
@@ -116,7 +115,9 @@ async def main() -> None:
 
             is_correct = pred_label == gold_label
             correct += int(is_correct)
-            print(f"[{i}] text={text!r}\n    gold={gold_label}\n    pred={pred} -> mapped={pred_label} {'✅' if is_correct else '❌'}")
+            print(
+                f"[{i}] text={text!r}\n    gold={gold_label}\n    pred={pred} -> mapped={pred_label} {'✅' if is_correct else '❌'}"
+            )
 
     if idxs:
         acc = correct / len(idxs)
@@ -137,7 +138,11 @@ async def main() -> None:
     with LMOverridesContext([{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]):
         _ = await classify_sample(lm_traced, test_text, label_names)
     # inspect trace
-    events = [e for e in (tracer.current_session.event_history if tracer.current_session else []) if isinstance(e, LMCAISEvent)]
+    events = [
+        e
+        for e in (tracer.current_session.event_history if tracer.current_session else [])
+        if isinstance(e, LMCAISEvent)
+    ]
     assert events, "No LMCAISEvent recorded by SessionTracer"
     cr = events[-1].call_records[0]
     traced_user = ""
@@ -145,7 +150,7 @@ async def main() -> None:
         if m.role == "user":
             for part in m.parts:
                 if getattr(part, "type", None) == "text":
-                    traced_user += (part.text or "")
+                    traced_user += part.text or ""
     assert "ATM" in traced_user, f"Expected substitution in traced prompt; got: {traced_user!r}"
     print("LM path trace verified: substitution present in traced prompt.")
     await tracer.end_timestep()
@@ -155,7 +160,7 @@ async def main() -> None:
     try:
         import synth_ai.lm.provider_support.openai as _synth_openai_patch  # noqa: F401
         from openai import AsyncOpenAI
-        from datasets import load_dataset as _ld  # ensure datasets present
+
         base_url = os.getenv("OPENAI_BASE_URL", "https://api.groq.com/openai/v1")
         api_key = os.getenv("OPENAI_API_KEY") or os.getenv("GROQ_API_KEY") or ""
         client = AsyncOpenAI(base_url=base_url, api_key=api_key)
@@ -163,8 +168,12 @@ async def main() -> None:
             {"role": "system", "content": "Echo user label."},
             {"role": "user", "content": f"Please classify: {test_text}"},
         ]
-        with LMOverridesContext([{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]):
-            resp = await client.chat.completions.create(model=model, messages=messages, temperature=0)
+        with LMOverridesContext(
+            [{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]
+        ):
+            _ = await client.chat.completions.create(
+                model=model, messages=messages, temperature=0
+            )
         # Not all models echo input; instead, verify that our injected expectation matches
         expected_user = messages[1]["content"].replace("atm", "ATM")
         if messages[1]["content"] == expected_user:
@@ -176,13 +185,16 @@ async def main() -> None:
 
     # 3) Anthropic wrapper path (AsyncClient): ensure apply_injection is active
     try:
-        import synth_ai.lm.provider_support.anthropic as _synth_anthropic_patch  # noqa: F401
         import anthropic
+        import synth_ai.lm.provider_support.anthropic as _synth_anthropic_patch  # noqa: F401
+
         a_model = os.getenv("ANTHROPIC_MODEL", "claude-3-5-haiku-20241022")
         a_key = os.getenv("ANTHROPIC_API_KEY")
         if a_key:
             a_client = anthropic.AsyncClient(api_key=a_key)
-            with LMOverridesContext([{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]):
+            with LMOverridesContext(
+                [{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]
+            ):
                 _ = await a_client.messages.create(
                     model=a_model,
                     system="Echo user label.",