synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/sdk/judging/client.py

@@ -0,0 +1,191 @@
+"""Experimental Judge API client.
+
+This surface is experimental and subject to change without notice.
+Set environment variable `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from typing import Any, Literal, TypedDict
+
+from synth_ai.core.http import AsyncHttpClient, HTTPError
+from synth_ai.core.tracing_v3.serialization import normalize_for_json
+from synth_ai.sdk.graphs import VerifierClient as GraphVerifierClient
+
+Provider = Literal["groq", "gemini"]
+
+
+class JudgeOptions(TypedDict, total=False):
+    event: bool
+    outcome: bool
+    rubric_id: str
+    rubric_overrides: dict[str, Any]
+    provider: Provider
+    model: str
+    max_concurrency: int
+    verifier_type: str
+
+
+class JudgeScoreResponse(TypedDict, total=False):
+    status: str
+    event_rewards: list[dict[str, Any]]
+    outcome_reward: dict[str, Any]
+    details: dict[str, Any]
+
+
+class JudgeClient:
+    """Legacy client for LLM-based evaluation of task app traces.
+    
+    This client provides programmatic access to Synth AI's judge API, which uses
+    LLMs to evaluate task execution traces and generate rewards. The judge can
+    evaluate both event-level (step-by-step) and outcome-level (episode-level) rewards.
+    
+    .. warning::
+        This API is experimental and subject to change without notice.
+        Set `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
+    
+    Example:
+        >>> from synth_ai.sdk.judging import JudgeClient, JudgeOptions
+        >>> 
+        >>> client = JudgeClient(
+        ...     base_url="https://api.usesynth.ai",
+        ...     api_key=os.environ["SYNTH_API_KEY"],
+        ... )
+        >>> 
+        >>> # Score a trace with outcome reward
+        >>> result = await client.score(
+        ...     trace=my_trace_dict,
+        ...     policy_name="my_policy",
+        ...     task_app_id="heartdisease",
+        ...     options=JudgeOptions(
+        ...         outcome=True,
+        ...         rubric_id="accuracy",
+        ...         provider="groq",
+        ...         model="llama-3.1-8b-instant",
+        ...     ),
+        ... )
+        >>> 
+        >>> print(f"Outcome reward: {result['outcome_reward']}")
+    """
+    
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None:
+        """Initialize the judge client.
+        
+        Args:
+            base_url: Base URL for the Synth AI API
+            api_key: API key for authentication
+            timeout: Request timeout in seconds (default: 60.0)
+        """
+        _silence = (os.getenv("SYNTH_SILENCE_EXPERIMENTAL") or "").strip().lower()
+        if _silence not in {"1", "true", "t", "yes", "y", "on"}:
+            warnings.warn(
+                "Legacy API: synth_ai.sdk.judging.JudgeClient is legacy. "
+                "Use synth_ai.sdk.graphs.VerifierClient or GraphCompletionsClient instead.",
+                UserWarning,
+                stacklevel=2,
+            )
+        self._base = base_url.rstrip("/")
+        self._key = api_key
+        self._timeout = timeout
+
+    async def score(
+        self,
+        *,
+        trace: dict[str, Any] | Any,
+        policy_name: str,
+        task_app_id: str,
+        options: JudgeOptions,
+        rubric: dict[str, Any] | None = None,
+        verifier_type: str | None = None,
+        task_app_base_url: str | None = None,
+    ) -> JudgeScoreResponse:
+        """Score a task execution trace using LLM-based evaluation.
+        
+        This method sends a trace to the judge API, which evaluates it according
+        to the provided rubric and returns event-level and/or outcome-level rewards.
+        
+        Args:
+            trace: Task execution trace (SessionTrace dict or compatible object)
+            policy_name: Name of the policy that generated this trace
+            task_app_id: Identifier for the task app (e.g., "heartdisease")
+            options: Judge configuration options:
+                - event: Whether to generate event-level rewards (default: False)
+                - outcome: Whether to generate outcome-level reward (default: False)
+                - rubric_id: Rubric identifier to use for evaluation
+                - rubric_overrides: Optional rubric modifications
+                - provider: LLM provider ("groq" or "gemini")
+                - model: Model identifier (e.g., "llama-3.1-8b-instant")
+                - max_concurrency: Max concurrent judge calls (default: 1)
+            rubric: Optional explicit rubric criteria (event/outcome lists)
+            verifier_type: Optional zero-shot verifier graph ID (e.g., "zero_shot_verifier_single")
+            task_app_base_url: Optional base URL for task app (for rubric fetching)
+            
+        Returns:
+            JudgeScoreResponse with:
+                - status: "ok" or error status
+                - event_rewards: List of event-level reward dicts (if event=True)
+                - outcome_reward: Outcome-level reward dict (if outcome=True)
+                - details: Additional evaluation details
+                
+        Raises:
+            ValueError: If validation fails or rubric is invalid
+            PermissionError: If authentication fails
+            FileNotFoundError: If task app or rubric not found
+            Exception: For rate limiting or transient errors
+        """
+        trace_payload = normalize_for_json(trace)
+        task_app_payload = {"id": task_app_id}
+        if task_app_base_url:
+            task_app_payload["base_url"] = task_app_base_url
+
+        selected_verifier = verifier_type or (options or {}).get("verifier_type")
+        if selected_verifier:
+            graph_input = {
+                "policy_name": policy_name,
+                "task_app": task_app_payload,
+                "session_trace": trace_payload,
+                "trace": trace_payload,
+                "options": options or {},
+            }
+            if rubric is not None:
+                graph_input["rubric"] = normalize_for_json(rubric)
+            body = {"job_id": selected_verifier, "input": graph_input}
+        else:
+            body = {
+                "policy_name": policy_name,
+                "task_app": task_app_payload,
+                "trace": trace_payload,
+                "options": options or {},
+            }
+            if rubric is not None:
+                body["rubric"] = normalize_for_json(rubric)
+        try:
+            async with AsyncHttpClient(self._base, self._key, timeout=self._timeout) as http:
+                if selected_verifier:
+                    js = await http.post_json("/api/graphs/completions", json=body)
+                    if isinstance(js, dict) and "output" in js:
+                        js = js["output"]
+                else:
+                    js = await http.post_json("/api/judge/v1/score", json=body)
+                if not isinstance(js, dict):
+                    raise ValueError("invalid_judge_response_shape")
+                return js  # type: ignore[return-value]
+        except HTTPError as err:  # map to friendlier exceptions
+            status = int(getattr(err, "status", 0) or 0)
+            if status in (400, 422):
+                raise ValueError(f"judge_validation_error: {err.detail}") from err
+            if status in (401, 403):
+                raise PermissionError(f"judge_auth_error: {err.detail}") from err
+            if status == 404:
+                raise FileNotFoundError(f"judge_route_not_found: {err.detail}") from err
+            if status == 429:
+                raise Exception("judge_rate_limited") from err  # replace with RetryLater in future
+            if status >= 500:
+                raise Exception("judge_transient_error") from err  # replace with TransientError in future
+            raise
+
+
+class VerifierClient(GraphVerifierClient):
+    """Deprecated alias for graph-based VerifierClient."""

synth_ai/sdk/judging/schemas.py

@@ -0,0 +1,222 @@
+"""
+Verifier API Contract Schemas
+
+These schemas define the expected structure for requests and responses
+to the verifier scoring endpoint at POST /api/judge/v1/score. Zero-shot
+verifier graphs use the same response format via POST /api/graphs/completions.
+
+This is the canonical contract that the backend MUST conform to.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class CriterionScorePayload(BaseModel):
+    """Per-criterion score returned by the judge."""
+
+    score: float = Field(..., description="Numeric score for this criterion")
+    reason: str = Field(default="", description="Explanation for the score")
+    weight: float = Field(default=1.0, description="Weight of this criterion")
+    description: str = Field(default="", description="Description of the criterion")
+
+
+class ReviewPayload(BaseModel):
+    """Rubric review (event-level or outcome-level)."""
+
+    criteria: dict[str, CriterionScorePayload] = Field(
+        default_factory=dict,
+        description="Map of criterion keys to their scores"
+    )
+    total: float = Field(default=0.0, description="Aggregated total score")
+    summary: Optional[str] = Field(None, description="Optional text summary")
+
+
+class JudgeScoreResponse(BaseModel):
+    """
+    Response body for POST /api/judge/v1/score.
+
+    This is the canonical contract that judge backends MUST return and is
+    also used as the zero-shot verifier graph output.
+    """
+
+    status: Literal["ok", "failed"] = Field(default="ok", description="Request status")
+    event_reviews: list[ReviewPayload] = Field(
+        default_factory=list,
+        description="List of per-event rubric reviews (one per step)"
+    )
+    outcome_review: Optional[ReviewPayload] = Field(
+        None,
+        description="Optional outcome-level rubric review"
+    )
+    event_totals: list[float] = Field(
+        default_factory=list,
+        description="List of aggregated scores per event (matches event_reviews length)"
+    )
+    details: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional details (provider, latency, etc.)"
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Request metadata (provider, options, etc.)"
+    )
+
+    def aggregate_event_reward(self) -> Optional[float]:
+        """
+        Aggregate all event totals into a single reward.
+        
+        Returns:
+            Sum of all event_totals, or None if empty
+        """
+        if not self.event_totals:
+            return None
+        return sum(self.event_totals)
+
+    def aggregate_outcome_reward(self) -> Optional[float]:
+        """
+        Extract outcome reward from outcome_review.
+        
+        Returns:
+            outcome_review.total, or None if no outcome review
+        """
+        if self.outcome_review is None:
+            return None
+        return self.outcome_review.total
+
+
+# Request schemas for completeness
+
+class JudgeTaskApp(BaseModel):
+    """Task application metadata."""
+    
+    id: str = Field(..., description="Task app identifier")
+    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
+
+
+class JudgeOptions(BaseModel):
+    """Judge provider and configuration options."""
+    
+    provider: Optional[str] = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: Optional[str] = Field(None, description="Model identifier")
+    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
+    event: bool = Field(True, description="Enable event-level judging")
+    outcome: bool = Field(True, description="Enable outcome-level judging")
+
+
+class JudgeTracePayload(BaseModel):
+    """Trace payload containing trajectory context."""
+    
+    event_history: list[dict[str, Any]] = Field(..., description="List of events/steps")
+    markov_blanket_message_history: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description="Optional message history for context"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
+
+
+class JudgeScoreRequest(BaseModel):
+    """Request body for POST /api/judge/v1/score."""
+    
+    policy_name: str = Field(..., description="Name of the policy being evaluated")
+    task_app: JudgeTaskApp = Field(..., description="Task application metadata")
+    trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
+    options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
+    rubric: Optional[dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
+
+
+# Verifier input validation schemas
+
+class CalibrationExampleInput(BaseModel):
+    """Input schema for a calibration example (few-shot verifier).
+    
+    Validates that the example has a valid trace and matching rewards.
+    Uses synth_ai.data.rewards.CalibrationExample dataclass for structure.
+    """
+    
+    session_trace: dict[str, Any] = Field(..., description="V3 SessionTrace format (validated separately)")
+    event_rewards: list[Annotated[float, Field(ge=0.0, le=1.0)]] = Field(
+        ..., 
+        description="List of rewards per event (0.0-1.0), must match number of events in trace"
+    )
+    outcome_reward: Annotated[float, Field(ge=0.0, le=1.0)] = Field(
+        ..., 
+        description="Overall outcome reward (0.0-1.0)"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Optional metadata")
+    
+    @model_validator(mode="after")
+    def validate_rewards_match_trace(self) -> "CalibrationExampleInput":
+        """Validate that event_rewards length matches trace events."""
+        # Count events in trace
+        trace_events = self._count_trace_events()
+        if len(self.event_rewards) != trace_events:
+            raise ValueError(
+                f"event_rewards length ({len(self.event_rewards)}) doesn't match trace events ({trace_events}). "
+                f"Each event in the trace must have a corresponding reward."
+            )
+        return self
+    
+    def _count_trace_events(self) -> int:
+        """Count total events in session_trace."""
+        count = 0
+        # Try event_history first (V3 format)
+        if isinstance(self.session_trace, dict):
+            event_history = self.session_trace.get("event_history", [])
+            if isinstance(event_history, list):
+                return len(event_history)
+            
+            # Try session_time_steps format
+            time_steps = self.session_trace.get("session_time_steps", [])
+            if isinstance(time_steps, list):
+                for step in time_steps:
+                    if isinstance(step, dict):
+                        events = step.get("events", [])
+                        if isinstance(events, list):
+                            count += len(events)
+                return count
+        return 0
+    
+    def to_dataclass(self) -> "CalibrationExample":
+        """Convert to synth_ai.data.rewards.CalibrationExample dataclass."""
+        from synth_ai.data.rewards import CalibrationExample
+        return CalibrationExample(
+            session_trace=self.session_trace,
+            event_rewards=self.event_rewards,
+            outcome_reward=self.outcome_reward,
+            metadata=self.metadata,
+        )
+
+
+class GoldExampleInput(BaseModel):
+    """Input schema for a gold example (contrastive verifier).
+    
+    Validates that the example has required fields with correct types.
+    Uses synth_ai.data.rewards.GoldExample dataclass for structure.
+    """
+    
+    summary: str = Field(..., min_length=1, description="Summary of the trace being evaluated")
+    gold_score: Annotated[float, Field(ge=0.0, le=1.0)] = Field(
+        ..., 
+        description="Gold-standard score (0.0-1.0)"
+    )
+    gold_reasoning: str = Field(..., min_length=1, description="Gold-standard reasoning/explanation")
+    session_trace: Optional[dict[str, Any]] = Field(
+        None, 
+        description="Optional full trace (for richer evaluation)"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Optional metadata")
+    
+    def to_dataclass(self) -> "GoldExample":
+        """Convert to synth_ai.data.rewards.GoldExample dataclass."""
+        from synth_ai.data.rewards import GoldExample
+        return GoldExample(
+            summary=self.summary,
+            gold_score=self.gold_score,
+            gold_reasoning=self.gold_reasoning,
+            session_trace=self.session_trace,
+            metadata=self.metadata,
+        )

synth_ai/sdk/judging/types.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Literal, TypedDict
+
+Track = Literal["process", "reasoning", "progress", "outcome"]
+
+
+class Judgement(TypedDict, total=False):
+	key: str
+	title: str
+	description: str
+	score: float
+	reason: str
+	confidence: float
+	scale: Literal["binary", "bounded", "count", "custom"]
+	source: dict
+
+
+class RewardJudgement(TypedDict, total=False):
+	judgement: Judgement
+	scope: Literal["step", "event", "outcome"]
+	turn: int | None
+	episode_id: str | None
+	reward_value: float | None
+	links: dict
+
+
+class TrackAggregate(TypedDict, total=False):
+	mean: float
+	median: float
+	std: float
+	n: int
+
+
+class RewardMetadata(TypedDict, total=False):
+	per_window: list[RewardJudgement]
+	aggregates: dict[Track, TrackAggregate]
+	overall: dict[str, float]  # {"final_outcome_score": float}
+	rubric: dict               # {"ids": {...}, "hash": "..."}
+	model_info: dict           # {"model": "...", ...}
+
+

synth_ai/sdk/learning/__init__.py

@@ -0,0 +1,69 @@
+from synth_ai.sdk.task import task_app_health, validate_task_app_url
+
+from .client import LearningClient
+from .health import backend_health, balance_autumn_normalized, pricing_preflight
+from .jobs import JobHandle, JobsApiResolver
+from .prompt_learning_client import (
+    PromptLearningClient,
+    get_prompt_text,
+    get_prompts,
+    get_scoring_summary,
+)
+from .rl import (
+    MAX_ENVIRONMENT_API_KEY_BYTES,
+    RlClient,
+    RLJobConfig,
+    RolloutEnvSpec,
+    RolloutMetrics,
+    RolloutPolicySpec,
+    RolloutRecordConfig,
+    RolloutRequest,
+    RolloutResponse,
+    RolloutSafetyConfig,
+    RolloutStep,
+    RolloutTrajectory,
+    encrypt_for_backend,
+    mint_environment_api_key,
+    setup_environment_api_key,
+)
+from .sft import FtClient
+from .sft.config import SFTJobConfig, prepare_sft_job_payload
+from .sse import stream_events as stream_job_events
+from .validators import validate_trainer_cfg_rl, validate_training_jsonl
+
+__all__ = [
+    "LearningClient",
+    "RlClient",
+    "RLJobConfig",
+    "FtClient",
+    "SFTJobConfig",
+    "prepare_sft_job_payload",
+    "PromptLearningClient",
+    "get_prompts",
+    "get_prompt_text",
+    "get_scoring_summary",
+    "RolloutEnvSpec",
+    "RolloutPolicySpec",
+    "RolloutRecordConfig",
+    "RolloutSafetyConfig",
+    "RolloutRequest",
+    "RolloutStep",
+    "RolloutTrajectory",
+    "RolloutMetrics",
+    "RolloutResponse",
+    "mint_environment_api_key",
+    "encrypt_for_backend",
+    "setup_environment_api_key",
+    "MAX_ENVIRONMENT_API_KEY_BYTES",
+    # convenience re-export for typing
+    "validate_training_jsonl",
+    "validate_trainer_cfg_rl",
+    "validate_task_app_url",
+    "backend_health",
+    "task_app_health",
+    "pricing_preflight",
+    "balance_autumn_normalized",
+    "stream_job_events",
+    "JobHandle",
+    "JobsApiResolver",
+]