synth-ai 0.2.8.dev2__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/sdk/judging/schemas.py

@@ -0,0 +1,222 @@
+"""
+Verifier API Contract Schemas
+
+These schemas define the expected structure for requests and responses
+to the verifier scoring endpoint at POST /api/judge/v1/score. Zero-shot
+verifier graphs use the same response format via POST /api/graphs/completions.
+
+This is the canonical contract that the backend MUST conform to.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class CriterionScorePayload(BaseModel):
+    """Per-criterion score returned by the judge."""
+
+    score: float = Field(..., description="Numeric score for this criterion")
+    reason: str = Field(default="", description="Explanation for the score")
+    weight: float = Field(default=1.0, description="Weight of this criterion")
+    description: str = Field(default="", description="Description of the criterion")
+
+
+class ReviewPayload(BaseModel):
+    """Rubric review (event-level or outcome-level)."""
+
+    criteria: dict[str, CriterionScorePayload] = Field(
+        default_factory=dict,
+        description="Map of criterion keys to their scores"
+    )
+    total: float = Field(default=0.0, description="Aggregated total score")
+    summary: Optional[str] = Field(None, description="Optional text summary")
+
+
+class JudgeScoreResponse(BaseModel):
+    """
+    Response body for POST /api/judge/v1/score.
+
+    This is the canonical contract that judge backends MUST return and is
+    also used as the zero-shot verifier graph output.
+    """
+
+    status: Literal["ok", "failed"] = Field(default="ok", description="Request status")
+    event_reviews: list[ReviewPayload] = Field(
+        default_factory=list,
+        description="List of per-event rubric reviews (one per step)"
+    )
+    outcome_review: Optional[ReviewPayload] = Field(
+        None,
+        description="Optional outcome-level rubric review"
+    )
+    event_totals: list[float] = Field(
+        default_factory=list,
+        description="List of aggregated scores per event (matches event_reviews length)"
+    )
+    details: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional details (provider, latency, etc.)"
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Request metadata (provider, options, etc.)"
+    )
+
+    def aggregate_event_reward(self) -> Optional[float]:
+        """
+        Aggregate all event totals into a single reward.
+        
+        Returns:
+            Sum of all event_totals, or None if empty
+        """
+        if not self.event_totals:
+            return None
+        return sum(self.event_totals)
+
+    def aggregate_outcome_reward(self) -> Optional[float]:
+        """
+        Extract outcome reward from outcome_review.
+        
+        Returns:
+            outcome_review.total, or None if no outcome review
+        """
+        if self.outcome_review is None:
+            return None
+        return self.outcome_review.total
+
+
+# Request schemas for completeness
+
+class JudgeTaskApp(BaseModel):
+    """Task application metadata."""
+    
+    id: str = Field(..., description="Task app identifier")
+    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
+
+
+class JudgeOptions(BaseModel):
+    """Judge provider and configuration options."""
+    
+    provider: Optional[str] = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: Optional[str] = Field(None, description="Model identifier")
+    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
+    event: bool = Field(True, description="Enable event-level judging")
+    outcome: bool = Field(True, description="Enable outcome-level judging")
+
+
+class JudgeTracePayload(BaseModel):
+    """Trace payload containing trajectory context."""
+    
+    event_history: list[dict[str, Any]] = Field(..., description="List of events/steps")
+    markov_blanket_message_history: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description="Optional message history for context"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
+
+
+class JudgeScoreRequest(BaseModel):
+    """Request body for POST /api/judge/v1/score."""
+    
+    policy_name: str = Field(..., description="Name of the policy being evaluated")
+    task_app: JudgeTaskApp = Field(..., description="Task application metadata")
+    trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
+    options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
+    rubric: Optional[dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
+
+
+# Verifier input validation schemas
+
+class CalibrationExampleInput(BaseModel):
+    """Input schema for a calibration example (few-shot verifier).
+    
+    Validates that the example has a valid trace and matching rewards.
+    Uses synth_ai.data.rewards.CalibrationExample dataclass for structure.
+    """
+    
+    session_trace: dict[str, Any] = Field(..., description="V3 SessionTrace format (validated separately)")
+    event_rewards: list[Annotated[float, Field(ge=0.0, le=1.0)]] = Field(
+        ..., 
+        description="List of rewards per event (0.0-1.0), must match number of events in trace"
+    )
+    outcome_reward: Annotated[float, Field(ge=0.0, le=1.0)] = Field(
+        ..., 
+        description="Overall outcome reward (0.0-1.0)"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Optional metadata")
+    
+    @model_validator(mode="after")
+    def validate_rewards_match_trace(self) -> "CalibrationExampleInput":
+        """Validate that event_rewards length matches trace events."""
+        # Count events in trace
+        trace_events = self._count_trace_events()
+        if len(self.event_rewards) != trace_events:
+            raise ValueError(
+                f"event_rewards length ({len(self.event_rewards)}) doesn't match trace events ({trace_events}). "
+                f"Each event in the trace must have a corresponding reward."
+            )
+        return self
+    
+    def _count_trace_events(self) -> int:
+        """Count total events in session_trace."""
+        count = 0
+        # Try event_history first (V3 format)
+        if isinstance(self.session_trace, dict):
+            event_history = self.session_trace.get("event_history", [])
+            if isinstance(event_history, list):
+                return len(event_history)
+            
+            # Try session_time_steps format
+            time_steps = self.session_trace.get("session_time_steps", [])
+            if isinstance(time_steps, list):
+                for step in time_steps:
+                    if isinstance(step, dict):
+                        events = step.get("events", [])
+                        if isinstance(events, list):
+                            count += len(events)
+                return count
+        return 0
+    
+    def to_dataclass(self) -> "CalibrationExample":
+        """Convert to synth_ai.data.rewards.CalibrationExample dataclass."""
+        from synth_ai.data.rewards import CalibrationExample
+        return CalibrationExample(
+            session_trace=self.session_trace,
+            event_rewards=self.event_rewards,
+            outcome_reward=self.outcome_reward,
+            metadata=self.metadata,
+        )
+
+
+class GoldExampleInput(BaseModel):
+    """Input schema for a gold example (contrastive verifier).
+    
+    Validates that the example has required fields with correct types.
+    Uses synth_ai.data.rewards.GoldExample dataclass for structure.
+    """
+    
+    summary: str = Field(..., min_length=1, description="Summary of the trace being evaluated")
+    gold_score: Annotated[float, Field(ge=0.0, le=1.0)] = Field(
+        ..., 
+        description="Gold-standard score (0.0-1.0)"
+    )
+    gold_reasoning: str = Field(..., min_length=1, description="Gold-standard reasoning/explanation")
+    session_trace: Optional[dict[str, Any]] = Field(
+        None, 
+        description="Optional full trace (for richer evaluation)"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Optional metadata")
+    
+    def to_dataclass(self) -> "GoldExample":
+        """Convert to synth_ai.data.rewards.GoldExample dataclass."""
+        from synth_ai.data.rewards import GoldExample
+        return GoldExample(
+            summary=self.summary,
+            gold_score=self.gold_score,
+            gold_reasoning=self.gold_reasoning,
+            session_trace=self.session_trace,
+            metadata=self.metadata,
+        )

synth_ai/sdk/judging/types.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Literal, TypedDict
+
+Track = Literal["process", "reasoning", "progress", "outcome"]
+
+
+class Judgement(TypedDict, total=False):
+	key: str
+	title: str
+	description: str
+	score: float
+	reason: str
+	confidence: float
+	scale: Literal["binary", "bounded", "count", "custom"]
+	source: dict
+
+
+class RewardJudgement(TypedDict, total=False):
+	judgement: Judgement
+	scope: Literal["step", "event", "outcome"]
+	turn: int | None
+	episode_id: str | None
+	reward_value: float | None
+	links: dict
+
+
+class TrackAggregate(TypedDict, total=False):
+	mean: float
+	median: float
+	std: float
+	n: int
+
+
+class RewardMetadata(TypedDict, total=False):
+	per_window: list[RewardJudgement]
+	aggregates: dict[Track, TrackAggregate]
+	overall: dict[str, float]  # {"final_outcome_score": float}
+	rubric: dict               # {"ids": {...}, "hash": "..."}
+	model_info: dict           # {"model": "...", ...}
+
+

synth_ai/sdk/learning/__init__.py

@@ -0,0 +1,99 @@
+from synth_ai.sdk.task import task_app_health, validate_task_app_url
+
+from .client import LearningClient
+from .context_learning_client import (
+    ContextLearningClient,
+    create_job as create_context_learning_job,
+    get_best_script as get_context_learning_best_script,
+    get_job_status as get_context_learning_status,
+    run_job as run_context_learning_job,
+)
+from .context_learning_types import (
+    AlgorithmConfig,
+    BestScriptResult,
+    ContextLearningEvent,
+    ContextLearningJobConfig,
+    ContextLearningJobStatus,
+    ContextLearningMetric,
+    ContextLearningResults,
+    EnvironmentConfig,
+)
+from .health import backend_health, balance_autumn_normalized, pricing_preflight
+from .jobs import JobHandle, JobsApiResolver
+from .prompt_learning_client import (
+    PromptLearningClient,
+    get_prompt_text,
+    get_prompts,
+    get_scoring_summary,
+)
+from .rl import (
+    MAX_ENVIRONMENT_API_KEY_BYTES,
+    RlClient,
+    RLJobConfig,
+    RolloutEnvSpec,
+    RolloutMetrics,
+    RolloutPolicySpec,
+    RolloutRecordConfig,
+    RolloutRequest,
+    RolloutResponse,
+    RolloutSafetyConfig,
+    encrypt_for_backend,
+    mint_environment_api_key,
+    setup_environment_api_key,
+)
+from .sft import FtClient
+from .sft.config import SFTJobConfig, prepare_sft_job_payload
+from .sse import stream_events as stream_job_events
+from .validators import validate_trainer_cfg_rl, validate_training_jsonl
+
+__all__ = [
+    # Learning clients
+    "LearningClient",
+    "RlClient",
+    "RLJobConfig",
+    "FtClient",
+    "SFTJobConfig",
+    "prepare_sft_job_payload",
+    # Prompt Learning
+    "PromptLearningClient",
+    "get_prompts",
+    "get_prompt_text",
+    "get_scoring_summary",
+    # Context Learning
+    "ContextLearningClient",
+    "ContextLearningJobConfig",
+    "ContextLearningJobStatus",
+    "ContextLearningEvent",
+    "ContextLearningMetric",
+    "ContextLearningResults",
+    "BestScriptResult",
+    "EnvironmentConfig",
+    "AlgorithmConfig",
+    "create_context_learning_job",
+    "get_context_learning_status",
+    "get_context_learning_best_script",
+    "run_context_learning_job",
+    # RL types
+    "RolloutEnvSpec",
+    "RolloutPolicySpec",
+    "RolloutRecordConfig",
+    "RolloutSafetyConfig",
+    "RolloutRequest",
+    "RolloutMetrics",
+    "RolloutResponse",
+    "mint_environment_api_key",
+    "encrypt_for_backend",
+    "setup_environment_api_key",
+    "MAX_ENVIRONMENT_API_KEY_BYTES",
+    # Utilities
+    "validate_training_jsonl",
+    "validate_trainer_cfg_rl",
+    "validate_task_app_url",
+    "backend_health",
+    "task_app_health",
+    "pricing_preflight",
+    "balance_autumn_normalized",
+    "stream_job_events",
+    "JobHandle",
+    "JobsApiResolver",
+]

synth_ai/sdk/learning/algorithms.py

@@ -0,0 +1,14 @@
+# class LearningModality(str, enum.Enum):
+#     """Modality of learning."""
+
+#     online_on_policy = "online_on_policy"
+#     online_off_policy = "online_off_policy"
+#     offline = "offline"
+
+
+# class LearningAlgorithm(str, enum.Enum):
+#     """Algorithm of learning."""
+
+#     gspo = "gspo"
+#     reinforce = "reinforce"
+#     sft = "sft"

synth_ai/{learning → sdk/learning}/client.py

@@ -1,9 +1,16 @@
 from __future__ import annotations
 
+from collections.abc import Callable
+from contextlib import suppress
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, TypedDict
 
-from ..http import AsyncHttpClient, HTTPError, sleep
+from synth_ai.core._utils.http import AsyncHttpClient, HTTPError, sleep
+from synth_ai.sdk.api.models.supported import (
+    UnsupportedModelError,
+    normalize_model_identifier,
+)
+from synth_ai.sdk.learning.sft.config import prepare_sft_job_payload
 
 
 class LearningClient:
@@ -20,7 +27,12 @@ class LearningClient:
             files = {"file": (p.name, content, _infer_content_type(p.name))}
             js = await http.post_multipart("/api/learning/files", data=data, files=files)
         if not isinstance(js, dict) or "id" not in js:
-            raise HTTPError(status=500, url="/api/learning/files", message="invalid_upload_response", body_snippet=str(js)[:200])
+            raise HTTPError(
+                status=500,
+                url="/api/learning/files",
+                message="invalid_upload_response",
+                body_snippet=str(js)[:200],
+            )
         return str(js["id"])
 
     async def create_job(
@@ -29,28 +41,56 @@ class LearningClient:
         training_type: str,
         model: str,
         training_file_id: str,
-        hyperparameters: Optional[Dict[str, Any]] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> Dict[str, Any]:
-        body = {
-            "training_type": training_type,
-            "model": model,
-            "training_file_id": training_file_id,
-            "hyperparameters": hyperparameters or {},
-            "metadata": metadata or {},
-        }
+        hyperparameters: dict[str, Any] | None = None,
+        metadata: dict[str, Any] | None = None,
+        validation_file: str | None = None,
+    ) -> dict[str, Any]:
+        lower_type = (training_type or "").strip().lower()
+        require_base = (
+            lower_type.startswith("sft")
+            or lower_type.startswith("fft")
+            or lower_type.startswith("qft")
+        )
+        try:
+            normalized_model = normalize_model_identifier(
+                model, allow_finetuned_prefixes=not require_base
+            )
+        except UnsupportedModelError as exc:
+            raise ValueError(str(exc)) from exc
+
+        if lower_type.startswith("sft") or lower_type in {"fft", "qft"}:
+            body = prepare_sft_job_payload(
+                model=model,
+                training_file=training_file_id,
+                hyperparameters=hyperparameters,
+                metadata=metadata,
+                training_type=training_type or "sft_offline",
+                validation_file=validation_file,
+                training_file_field="training_file_id",
+                require_training_file=True,
+            )
+        else:
+            body = {
+                "training_type": training_type,
+                "model": normalized_model,
+                "training_file_id": training_file_id,
+                "hyperparameters": hyperparameters or {},
+                "metadata": metadata or {},
+            }
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             return await http.post_json("/api/learning/jobs", json=body)
 
-    async def start_job(self, job_id: str) -> Dict[str, Any]:
+    async def start_job(self, job_id: str) -> dict[str, Any]:
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             return await http.post_json(f"/api/learning/jobs/{job_id}/start", json={})
 
-    async def get_job(self, job_id: str) -> Dict[str, Any]:
+    async def get_job(self, job_id: str) -> dict[str, Any]:
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             return await http.get(f"/api/learning/jobs/{job_id}")
 
-    async def get_events(self, job_id: str, *, since_seq: int = 0, limit: int = 200) -> List[Dict[str, Any]]:
+    async def get_events(
+        self, job_id: str, *, since_seq: int = 0, limit: int = 200
+    ) -> list[dict[str, Any]]:
         params = {"since_seq": since_seq, "limit": limit}
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             js = await http.get(f"/api/learning/jobs/{job_id}/events", params=params)
@@ -58,8 +98,16 @@ class LearningClient:
             return js["events"]
         return []
 
-    async def get_metrics(self, job_id: str, *, name: str | None = None, after_step: int | None = None, limit: int = 500, run_id: str | None = None) -> List[Dict[str, Any]]:
-        params: Dict[str, Any] = {"limit": limit}
+    async def get_metrics(
+        self,
+        job_id: str,
+        *,
+        name: str | None = None,
+        after_step: int | None = None,
+        limit: int = 500,
+        run_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        params: dict[str, Any] = {"limit": limit}
         if name is not None:
             params["name"] = name
         if after_step is not None:
@@ -72,7 +120,7 @@ class LearningClient:
             return js["points"]
         return []
 
-    async def get_timeline(self, job_id: str, *, limit: int = 200) -> List[Dict[str, Any]]:
+    async def get_timeline(self, job_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
         params = {"limit": limit}
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             js = await http.get(f"/api/learning/jobs/{job_id}/timeline", params=params)
@@ -86,8 +134,8 @@ class LearningClient:
         *,
         interval_seconds: float = 2.0,
         max_seconds: float | None = 3600,
-        on_event: Callable[[Dict[str, Any]], None] | None = None,
-    ) -> Dict[str, Any]:
+        on_event: Callable[[dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
         last_seq = 0
         elapsed = 0.0
         while True:
@@ -97,10 +145,8 @@ class LearningClient:
                 if isinstance(e, dict) and isinstance(e.get("seq"), int):
                     last_seq = max(last_seq, int(e["seq"]))
                 if on_event:
-                    try:
+                    with suppress(Exception):
                         on_event(e)
-                    except Exception:
-                        pass
 
             # Status
             job = await self.get_job(job_id)
@@ -115,7 +161,9 @@ class LearningClient:
                 raise TimeoutError(f"Polling timed out after {elapsed} seconds for job {job_id}")
 
     # --- Optional diagnostics ---
-    async def pricing_preflight(self, *, job_type: str, gpu_type: str, estimated_seconds: float, container_count: int) -> Dict[str, Any]:
+    async def pricing_preflight(
+        self, *, job_type: str, gpu_type: str, estimated_seconds: float, container_count: int
+    ) -> dict[str, Any]:
         body = {
             "job_type": job_type,
             "gpu_type": gpu_type,
@@ -125,17 +173,62 @@ class LearningClient:
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             js = await http.post_json("/api/v1/pricing/preflight", json=body)
         if not isinstance(js, dict):
-            raise HTTPError(status=500, url="/api/v1/pricing/preflight", message="invalid_preflight_response", body_snippet=str(js)[:200])
+            raise HTTPError(
+                status=500,
+                url="/api/v1/pricing/preflight",
+                message="invalid_preflight_response",
+                body_snippet=str(js)[:200],
+            )
         return js
 
-    async def balance_autumn_normalized(self) -> Dict[str, Any]:
+    async def balance_autumn_normalized(self) -> dict[str, Any]:
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
             js = await http.get("/api/v1/balance/autumn-normalized")
         if not isinstance(js, dict):
-            raise HTTPError(status=500, url="/api/v1/balance/autumn-normalized", message="invalid_balance_response", body_snippet=str(js)[:200])
+            raise HTTPError(
+                status=500,
+                url="/api/v1/balance/autumn-normalized",
+                message="invalid_balance_response",
+                body_snippet=str(js)[:200],
+            )
         return js
 
 
+class FineTunedModelInfo(TypedDict, total=False):
+    id: str
+    base_model: str | None
+    created_at: int | None
+    job_id: str | None
+    status: str | None
+
+
+class LearningClient(LearningClient):  # type: ignore[misc]
+    async def list_fine_tuned_models(self) -> list[FineTunedModelInfo]:
+        """Return completed fine‑tuned models for the caller's organization.
+
+        Calls backend route `/api/learning/models` and returns a compact list.
+        """
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            js = await http.get("/api/learning/models")
+        if isinstance(js, dict) and isinstance(js.get("data"), list):
+            out: list[FineTunedModelInfo] = []
+            for item in js["data"]:
+                if not isinstance(item, dict):
+                    continue
+                rec: FineTunedModelInfo = {
+                    "id": str(item.get("id")),
+                    "base_model": item.get("base_model"),
+                    "created_at": item.get("created_at"),
+                    "job_id": item.get("job_id"),
+                    "status": item.get("status"),
+                }
+                if rec.get("id"):
+                    out.append(rec)
+            return out
+        # Fallback: empty list on unexpected shape
+        return []
+
+
 def _infer_content_type(filename: str) -> str:
     name = filename.lower()
     if name.endswith(".jsonl"):
@@ -145,5 +238,3 @@ def _infer_content_type(filename: str) -> str:
     if name.endswith(".txt"):
         return "text/plain"
     return "application/octet-stream"
-
-

synth_ai/sdk/learning/config.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .rl.config import RLJobConfig
+
+__all__ = ["RLJobConfig"]

synth_ai/{learning → sdk/learning}/constants.py

@@ -25,5 +25,3 @@ TERMINAL_EVENT_FAILURE = {
     "rl.job.failed",
     "workflow.failed",
 }
-
-