synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/learning/context_learning_types.py

@@ -0,0 +1,294 @@
+"""Type definitions for context learning data structures.
+
+Context Learning optimizes environment setup scripts (pre-flight/post-flight bash)
+for terminal/coding agents, similar to how prompt learning optimizes prompts.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class EnvironmentConfig:
+    """Environment configuration with pre-flight and post-flight scripts."""
+    
+    preflight_script: Optional[str] = None
+    postflight_script: Optional[str] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> EnvironmentConfig:
+        """Create an EnvironmentConfig from a dictionary."""
+        return cls(
+            preflight_script=data.get("preflight_script"),
+            postflight_script=data.get("postflight_script"),
+        )
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for API requests."""
+        return {
+            "preflight_script": self.preflight_script,
+            "postflight_script": self.postflight_script,
+        }
+
+
+@dataclass
+class AlgorithmConfig:
+    """Algorithm configuration for context learning optimization."""
+    
+    initial_population_size: int = 10
+    num_generations: int = 5
+    children_per_generation: int = 5
+    mutation_llm_model: Optional[str] = None
+    mutation_llm_provider: str = "openai"
+    policy_config: Optional[Dict[str, Any]] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> AlgorithmConfig:
+        """Create an AlgorithmConfig from a dictionary."""
+        return cls(
+            initial_population_size=data.get("initial_population_size", 10),
+            num_generations=data.get("num_generations", 5),
+            children_per_generation=data.get("children_per_generation", 5),
+            mutation_llm_model=data.get("mutation_llm_model"),
+            mutation_llm_provider=data.get("mutation_llm_provider", "openai"),
+            policy_config=data.get("policy_config"),
+        )
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for API requests."""
+        result: Dict[str, Any] = {
+            "initial_population_size": self.initial_population_size,
+            "num_generations": self.num_generations,
+            "children_per_generation": self.children_per_generation,
+            "mutation_llm_provider": self.mutation_llm_provider,
+        }
+        if self.mutation_llm_model:
+            result["mutation_llm_model"] = self.mutation_llm_model
+        if self.policy_config:
+            result["policy_config"] = self.policy_config
+        return result
+
+
+@dataclass
+class ContextLearningJobConfig:
+    """Configuration for creating a context learning job."""
+    
+    task_app_url: str
+    evaluation_seeds: List[int]
+    task_app_api_key: Optional[str] = None
+    environment: Optional[EnvironmentConfig] = None
+    algorithm_config: Optional[AlgorithmConfig] = None
+    metadata: Optional[Dict[str, Any]] = None
+    org_id: Optional[str] = None
+    max_concurrent_rollouts: int = 10
+    verifier_base_url: Optional[str] = None
+    verifier_job_id: str = "zero_shot_verifier_rubric_mapreduce"
+    verifier_model: str = "gpt-4.1-mini"
+    require_agent_trace_log: bool = True
+    auto_start: bool = True
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> ContextLearningJobConfig:
+        """Create a ContextLearningJobConfig from a dictionary."""
+        env_data = data.get("environment")
+        env = EnvironmentConfig.from_dict(env_data) if isinstance(env_data, dict) else None
+        
+        algo_data = data.get("algorithm_config")
+        algo = AlgorithmConfig.from_dict(algo_data) if isinstance(algo_data, dict) else None
+        
+        return cls(
+            task_app_url=data["task_app_url"],
+            evaluation_seeds=data.get("evaluation_seeds", []),
+            task_app_api_key=data.get("task_app_api_key"),
+            environment=env,
+            algorithm_config=algo,
+            metadata=data.get("metadata"),
+            org_id=data.get("org_id"),
+            max_concurrent_rollouts=data.get("max_concurrent_rollouts", 10),
+            verifier_base_url=data.get("verifier_base_url"),
+            verifier_job_id=data.get("verifier_job_id", "zero_shot_verifier_rubric_mapreduce"),
+            verifier_model=data.get("verifier_model", "gpt-4.1-mini"),
+            require_agent_trace_log=data.get("require_agent_trace_log", True),
+            auto_start=data.get("auto_start", True),
+        )
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for API requests."""
+        result: Dict[str, Any] = {
+            "task_app_url": self.task_app_url,
+            "evaluation_seeds": self.evaluation_seeds,
+            "max_concurrent_rollouts": self.max_concurrent_rollouts,
+            "verifier_job_id": self.verifier_job_id,
+            "verifier_model": self.verifier_model,
+            "require_agent_trace_log": self.require_agent_trace_log,
+            "auto_start": self.auto_start,
+        }
+        if self.task_app_api_key:
+            result["task_app_api_key"] = self.task_app_api_key
+        if self.environment:
+            result["environment"] = self.environment.to_dict()
+        if self.algorithm_config:
+            result["algorithm_config"] = self.algorithm_config.to_dict()
+        if self.metadata:
+            result["metadata"] = self.metadata
+        if self.org_id:
+            result["org_id"] = self.org_id
+        if self.verifier_base_url:
+            result["verifier_base_url"] = self.verifier_base_url
+        return result
+
+
+@dataclass
+class ContextLearningJobStatus:
+    """Status of a context learning job."""
+    
+    job_id: str
+    status: str
+    created_at: str
+    started_at: Optional[str] = None
+    finished_at: Optional[str] = None
+    best_score: Optional[float] = None
+    best_preflight_script: Optional[str] = None
+    environment: Optional[Dict[str, Any]] = None
+    metadata: Optional[Dict[str, Any]] = None
+    recent_events: List[Dict[str, Any]] = field(default_factory=list)
+    error: Optional[str] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> ContextLearningJobStatus:
+        """Create a ContextLearningJobStatus from a dictionary."""
+        return cls(
+            job_id=data["job_id"],
+            status=data.get("status", "unknown"),
+            created_at=data.get("created_at", ""),
+            started_at=data.get("started_at"),
+            finished_at=data.get("finished_at"),
+            best_score=data.get("best_score"),
+            best_preflight_script=data.get("best_preflight_script"),
+            environment=data.get("environment"),
+            metadata=data.get("metadata"),
+            recent_events=data.get("recent_events", []),
+            error=data.get("error"),
+        )
+    
+    @property
+    def is_terminal(self) -> bool:
+        """Check if the job is in a terminal state."""
+        return self.status in {"completed", "succeeded", "failed", "cancelled"}
+    
+    @property
+    def is_successful(self) -> bool:
+        """Check if the job completed successfully."""
+        return self.status in {"completed", "succeeded"}
+
+
+@dataclass
+class ContextLearningEvent:
+    """An event from a context learning job."""
+    
+    event_type: str
+    message: str
+    timestamp: str
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    seq: Optional[int] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> ContextLearningEvent:
+        """Create a ContextLearningEvent from a dictionary."""
+        return cls(
+            event_type=data.get("event_type", data.get("type", "")),
+            message=data.get("message", ""),
+            timestamp=data.get("timestamp", data.get("created_at", "")),
+            metadata=data.get("metadata", data.get("data", {})),
+            seq=data.get("seq"),
+        )
+
+
+@dataclass
+class BestScriptResult:
+    """Result containing the best performing pre-flight script."""
+    
+    job_id: str
+    best_score: float
+    preflight_script: str
+    generation: int
+    variation_id: str
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> BestScriptResult:
+        """Create a BestScriptResult from a dictionary."""
+        return cls(
+            job_id=data.get("job_id", ""),
+            best_score=float(data.get("best_score", 0.0)),
+            preflight_script=str(data.get("preflight_script", "")),
+            generation=int(data.get("generation", 0)),
+            variation_id=str(data.get("variation_id", "")),
+            metadata=data.get("metadata", {}),
+        )
+
+
+@dataclass
+class ContextLearningMetric:
+    """A metric point from context learning optimization."""
+    
+    name: str
+    value: float
+    step: int
+    timestamp: str
+    data: Dict[str, Any] = field(default_factory=dict)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> ContextLearningMetric:
+        """Create a ContextLearningMetric from a dictionary."""
+        return cls(
+            name=data.get("name", ""),
+            value=float(data.get("value", 0.0)),
+            step=int(data.get("step", 0)),
+            timestamp=data.get("timestamp", data.get("created_at", "")),
+            data=data.get("data", {}),
+        )
+
+
+@dataclass
+class ContextLearningResults:
+    """Complete results from a context learning job."""
+    
+    job_id: str
+    status: str
+    best_score: Optional[float] = None
+    best_script: Optional[BestScriptResult] = None
+    generations_completed: int = 0
+    events: List[ContextLearningEvent] = field(default_factory=list)
+    metrics: List[ContextLearningMetric] = field(default_factory=list)
+    
+    @classmethod
+    def from_status_and_events(
+        cls,
+        status: ContextLearningJobStatus,
+        events: List[ContextLearningEvent],
+        best_script: Optional[BestScriptResult] = None,
+    ) -> ContextLearningResults:
+        """Create results from status and events."""
+        # Extract generation count from events
+        generations_completed = 0
+        for event in events:
+            if event.event_type == "context.learning.generation.completed":
+                gen = event.metadata.get("generation", 0)
+                if isinstance(gen, int) and gen > generations_completed:
+                    generations_completed = gen
+        
+        return cls(
+            job_id=status.job_id,
+            status=status.status,
+            best_score=status.best_score,
+            best_script=best_script,
+            generations_completed=generations_completed,
+            events=events,
+        )
+
+
+
+

synth_ai/sdk/learning/ft_client.py

@@ -0,0 +1,7 @@
+"""Backward-compatible shim for FtClient (moved to synth_ai.sdk.learning.sft.client)."""
+
+from __future__ import annotations
+
+from .sft.client import FtClient
+
+__all__ = ["FtClient"]

synth_ai/sdk/learning/health.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from typing import Any
+
+from synth_ai.core._utils.http import AsyncHttpClient
+
+
+def _api_base(b: str) -> str:
+    b = (b or "").rstrip("/")
+    return b if b.endswith("/api") else f"{b}/api"
+
+
+async def backend_health(base_url: str, api_key: str) -> dict[str, Any]:
+    async with AsyncHttpClient(base_url, api_key, timeout=15.0) as http:
+        js = await http.get(f"{_api_base(base_url)}/health")
+    return {"ok": True, "raw": js}
+
+
+async def task_app_health(task_app_url: str) -> dict[str, Any]:
+    # Delegate to central task module for consistency
+    from synth_ai.sdk.task.health import task_app_health as _th
+
+    return await _th(task_app_url)
+
+
+async def pricing_preflight(
+    base_url: str,
+    api_key: str,
+    *,
+    job_type: str,
+    gpu_type: str,
+    estimated_seconds: float,
+    container_count: int,
+) -> dict[str, Any]:
+    body = {
+        "job_type": job_type,
+        "gpu_type": gpu_type,
+        "estimated_seconds": float(estimated_seconds or 0.0),
+        "container_count": int(container_count or 1),
+    }
+    async with AsyncHttpClient(base_url, api_key, timeout=30.0) as http:
+        js = await http.post_json(f"{_api_base(base_url)}/v1/pricing/preflight", json=body)
+    return js if isinstance(js, dict) else {"raw": js}
+
+
+async def balance_autumn_normalized(base_url: str, api_key: str) -> dict[str, Any]:
+    async with AsyncHttpClient(base_url, api_key, timeout=30.0) as http:
+        js = await http.get(f"{_api_base(base_url)}/v1/balance/autumn-normalized")
+    return js if isinstance(js, dict) else {"raw": js}

synth_ai/sdk/learning/jobs.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from contextlib import suppress
+from typing import Any
+
+from synth_ai.core._utils.http import AsyncHttpClient, sleep
+
+from .constants import TERMINAL_EVENT_FAILURE, TERMINAL_EVENT_SUCCESS, TERMINAL_STATUSES
+
+
+def _api_base(b: str) -> str:
+    b = (b or "").rstrip("/")
+    return b if b.endswith("/api") else f"{b}/api"
+
+
+class JobsApiResolver:
+    def __init__(self, base_url: str, *, strict: bool) -> None:
+        self._base = _api_base(base_url)
+        self._strict = strict
+
+    def status_urls(self, job_id: str) -> list[str]:
+        if self._strict:
+            return [f"{self._base}/learning/jobs/{job_id}"]
+        return [
+            f"{self._base}/learning/jobs/{job_id}",
+            f"{self._base}/rl/jobs/{job_id}",
+            f"{self._base}/orchestration/jobs/{job_id}",
+        ]
+
+    def events_urls(self, job_id: str, since: int) -> list[str]:
+        if self._strict:
+            return [f"{self._base}/learning/jobs/{job_id}/events?since_seq={since}&limit=200"]
+        return [
+            f"{self._base}/learning/jobs/{job_id}/events?since_seq={since}&limit=200",
+            f"{self._base}/orchestration/jobs/{job_id}/events?since_seq={since}&limit=200",
+            # RL /jobs/{id}/events is SSE in backend; avoid in JSON poller
+        ]
+
+    def metrics_url(self, job_id: str, after_step: int) -> str:
+        return f"{self._base}/learning/jobs/{job_id}/metrics?after_step={after_step}&limit=200"
+
+
+class JobHandle:
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        job_id: str,
+        *,
+        strict: bool = True,
+        timeout: float = 600.0,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.job_id = job_id
+        self.strict = strict
+        self.timeout = timeout
+
+    async def poll_until_terminal(
+        self,
+        *,
+        interval_seconds: float = 2.0,
+        max_seconds: float | None = None,
+        empty_polls_threshold: int = 5,
+        startup_deadline_s: int = 45,
+        on_event: Callable[[dict[str, Any]], None] | None = None,
+        on_metric: Callable[[dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        last_seq_by_stream: dict[str, int] = {}
+        events_job_id: str | None = None
+        last_status: str | None = None
+        last_step_by_name: dict[str, int] = {}
+        empty_polls = 0
+        saw_any_event = False
+        start_t = time.time()
+        resolver = JobsApiResolver(self.base_url, strict=self.strict)
+        detected_fine_tuned_model: str | None = None
+
+        async with AsyncHttpClient(self.base_url, self.api_key, timeout=self.timeout) as http:
+            while True:
+                # Status
+                status_data: dict[str, Any] | None = None
+                for su in resolver.status_urls(self.job_id):
+                    try:
+                        status_data = await http.get(su)
+                        if isinstance(status_data, dict):
+                            break
+                    except Exception:
+                        continue
+                status = str((status_data or {}).get("status") or "").lower()
+                if status_data:
+                    linked = status_data.get("linked_job_id")
+                    if isinstance(linked, str) and linked and linked != events_job_id:
+                        events_job_id = linked
+                    # Capture fine_tuned_model if already present on status
+                    if not detected_fine_tuned_model:
+                        ftm = status_data.get("fine_tuned_model")
+                        if isinstance(ftm, str) and ftm:
+                            detected_fine_tuned_model = ftm
+                if status and status != last_status:
+                    last_status = status
+                    if on_event:
+                        with suppress(Exception):
+                            on_event({"type": "job.status", "message": status})
+
+                # Events
+                stream_ids = [self.job_id]
+                if events_job_id and events_job_id not in stream_ids:
+                    stream_ids.append(events_job_id)
+                total_events_this_cycle = 0
+                terminal_event_seen = False
+                terminal_event_status: str | None = None
+                for ev_id in stream_ids:
+                    since = last_seq_by_stream.get(ev_id, 0)
+                    for eu in resolver.events_urls(ev_id, since):
+                        try:
+                            ev_js = await http.get(eu)
+                        except Exception:
+                            continue
+                        events = (ev_js or {}).get("events") or (ev_js or {}).get("data") or []
+                        if not isinstance(events, list):
+                            events = []
+                        total_events_this_cycle += len(events)
+                        if events:
+                            saw_any_event = True
+                        for e in events:
+                            seq_val = int(e.get("seq") or 0)
+                            if seq_val <= last_seq_by_stream.get(ev_id, 0):
+                                continue
+                            last_seq_by_stream[ev_id] = seq_val
+                            if on_event:
+                                with suppress(Exception):
+                                    on_event(e)
+                            et = str(e.get("type") or e.get("event_type") or "").lower()
+                            # Capture fine_tuned_model from event data when available
+                            if not detected_fine_tuned_model:
+                                data_obj = e.get("data") or {}
+                                if isinstance(data_obj, dict):
+                                    ftm = data_obj.get("fine_tuned_model")
+                                    if isinstance(ftm, str) and ftm:
+                                        detected_fine_tuned_model = ftm
+                            if et in TERMINAL_EVENT_SUCCESS:
+                                terminal_event_seen = True
+                                terminal_event_status = "succeeded"
+                            elif et in TERMINAL_EVENT_FAILURE:
+                                terminal_event_seen = True
+                                terminal_event_status = "failed"
+
+                # Metrics
+                try:
+                    after = max(last_step_by_name.values()) if last_step_by_name else -1
+                    mu = resolver.metrics_url(self.job_id, after)
+                    md = await http.get(mu)
+                    for p in (md or {}).get("points", []):
+                        name = str(p.get("name") or "")
+                        step = int(p.get("step") or -1)
+                        if step <= last_step_by_name.get(name, -1):
+                            continue
+                        last_step_by_name[name] = step
+                        if on_metric:
+                            with suppress(Exception):
+                                on_metric(p)
+                except Exception:
+                    pass
+
+                # Terminal decisions
+                if terminal_event_seen or (status and status in TERMINAL_STATUSES):
+                    # Best-effort enrichment of final result with fine_tuned_model
+                    result_status = terminal_event_status or status or "completed"
+                    final_res: dict[str, Any] = {"status": result_status, "job_id": self.job_id}
+                    if not detected_fine_tuned_model:
+                        # Briefly try to re-fetch status to see if fine_tuned_model is persisted
+                        try:
+                            for su in resolver.status_urls(self.job_id):
+                                final_status = await http.get(su)
+                                if isinstance(final_status, dict):
+                                    ftm2 = final_status.get("fine_tuned_model")
+                                    if isinstance(ftm2, str) and ftm2:
+                                        detected_fine_tuned_model = ftm2
+                                        break
+                        except Exception:
+                            pass
+                    if detected_fine_tuned_model:
+                        final_res["fine_tuned_model"] = detected_fine_tuned_model
+                    return final_res
+
+                # Guards (relaxed): do not abort on consecutive empty polls
+                if total_events_this_cycle == 0:
+                    empty_polls += 1
+                else:
+                    empty_polls = 0
+                if not saw_any_event and (time.time() - start_t) > int(startup_deadline_s):
+                    raise AssertionError(
+                        f"No events observed within startup window ({startup_deadline_s}s). Investigate event streaming."
+                    )
+                await sleep(interval_seconds)
+                if max_seconds is not None and (time.time() - start_t) >= max_seconds:
+                    raise TimeoutError(
+                        f"Polling timed out after {max_seconds}s for job {self.job_id}"
+                    )