synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/graphs/verifier_schemas.py

@@ -0,0 +1,222 @@
+"""
+Verifier API Contract Schemas
+
+These schemas define the expected structure for requests and responses
+to the verifier scoring endpoint at POST /api/graphs/verifiers/completions.
+Zero-shot verifier graphs use the same response format via POST /api/graphs/completions.
+
+This is the canonical contract that the backend MUST conform to.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Any, Literal, Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class CriterionScorePayload(BaseModel):
+    """Per-criterion score returned by the verifier."""
+
+    score: float = Field(..., description="Numeric score for this criterion")
+    reason: str = Field(default="", description="Explanation for the score")
+    weight: float = Field(default=1.0, description="Weight of this criterion")
+    description: str = Field(default="", description="Description of the criterion")
+
+
+class ReviewPayload(BaseModel):
+    """Rubric review (event-level or outcome-level)."""
+
+    criteria: dict[str, CriterionScorePayload] = Field(
+        default_factory=dict,
+        description="Map of criterion keys to their scores"
+    )
+    total: float = Field(default=0.0, description="Aggregated total score")
+    summary: Optional[str] = Field(None, description="Optional text summary")
+
+
+class VerifierScoreResponse(BaseModel):
+    """
+    Response body for POST /api/graphs/verifiers/completions.
+
+    This is the canonical contract that verifier backends MUST return and is
+    also used as the zero-shot verifier graph output.
+    """
+
+    status: Literal["ok", "failed"] = Field(default="ok", description="Request status")
+    event_reviews: list[ReviewPayload] = Field(
+        default_factory=list,
+        description="List of per-event rubric reviews (one per step)"
+    )
+    outcome_review: Optional[ReviewPayload] = Field(
+        None,
+        description="Optional outcome-level rubric review"
+    )
+    event_totals: list[float] = Field(
+        default_factory=list,
+        description="List of aggregated scores per event (matches event_reviews length)"
+    )
+    details: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional details (provider, latency, etc.)"
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Request metadata (provider, options, etc.)"
+    )
+
+    def aggregate_event_reward(self) -> Optional[float]:
+        """
+        Aggregate all event totals into a single reward.
+        
+        Returns:
+            Sum of all event_totals, or None if empty
+        """
+        if not self.event_totals:
+            return None
+        return sum(self.event_totals)
+
+    def aggregate_outcome_reward(self) -> Optional[float]:
+        """
+        Extract outcome reward from outcome_review.
+        
+        Returns:
+            outcome_review.total, or None if no outcome review
+        """
+        if self.outcome_review is None:
+            return None
+        return self.outcome_review.total
+
+
+# Request schemas for completeness
+
+class VerifierTaskApp(BaseModel):
+    """Task application metadata."""
+    
+    id: str = Field(..., description="Task app identifier")
+    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
+
+
+class VerifierOptions(BaseModel):
+    """Verifier provider and configuration options."""
+    
+    provider: Optional[str] = Field(None, description="Verifier provider (e.g., 'openai', 'groq')")
+    model: Optional[str] = Field(None, description="Model identifier")
+    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
+    event: bool = Field(True, description="Enable event-level verification")
+    outcome: bool = Field(True, description="Enable outcome-level verification")
+
+
+class VerifierTracePayload(BaseModel):
+    """Trace payload containing trajectory context."""
+    
+    event_history: list[dict[str, Any]] = Field(..., description="List of events/steps")
+    markov_blanket_message_history: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description="Optional message history for context"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
+
+
+class VerifierScoreRequest(BaseModel):
+    """Request body for POST /api/graphs/verifiers/completions."""
+    
+    policy_name: str = Field(..., description="Name of the policy being evaluated")
+    task_app: VerifierTaskApp = Field(..., description="Task application metadata")
+    trace: VerifierTracePayload = Field(..., description="Trajectory trace to evaluate")
+    options: VerifierOptions = Field(default_factory=lambda: VerifierOptions(), description="Verifier options")
+    rubric: Optional[dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
+
+
+# Verifier input validation schemas
+
+class CalibrationExampleInput(BaseModel):
+    """Input schema for a calibration example (few-shot verifier).
+    
+    Validates that the example has a valid trace and matching rewards.
+    Uses synth_ai.data.rewards.CalibrationExample dataclass for structure.
+    """
+    
+    session_trace: dict[str, Any] = Field(..., description="V3 SessionTrace format (validated separately)")
+    event_rewards: list[Annotated[float, Field(ge=0.0, le=1.0)]] = Field(
+        ...,
+        description="List of rewards per event (0.0-1.0), must match number of events in trace"
+    )
+    outcome_reward: Annotated[float, Field(ge=0.0, le=1.0)] = Field(
+        ...,
+        description="Overall outcome reward (0.0-1.0)"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Optional metadata")
+    
+    @model_validator(mode="after")
+    def validate_rewards_match_trace(self) -> "CalibrationExampleInput":
+        """Validate that event_rewards length matches trace events."""
+        # Count events in trace
+        trace_events = self._count_trace_events()
+        if len(self.event_rewards) != trace_events:
+            raise ValueError(
+                f"event_rewards length ({len(self.event_rewards)}) doesn't match trace events ({trace_events}). "
+                f"Each event in the trace must have a corresponding reward."
+            )
+        return self
+    
+    def _count_trace_events(self) -> int:
+        """Count total events in session_trace."""
+        count = 0
+        # Try event_history first (V3 format)
+        if isinstance(self.session_trace, dict):
+            event_history = self.session_trace.get("event_history", [])
+            if isinstance(event_history, list):
+                return len(event_history)
+            
+            # Try session_time_steps format
+            time_steps = self.session_trace.get("session_time_steps", [])
+            if isinstance(time_steps, list):
+                for step in time_steps:
+                    if isinstance(step, dict):
+                        events = step.get("events", [])
+                        if isinstance(events, list):
+                            count += len(events)
+                return count
+        return 0
+    
+    def to_dataclass(self) -> "CalibrationExample":
+        """Convert to synth_ai.data.rewards.CalibrationExample dataclass."""
+        from synth_ai.data.rewards import CalibrationExample
+        return CalibrationExample(
+            session_trace=self.session_trace,
+            event_rewards=self.event_rewards,
+            outcome_reward=self.outcome_reward,
+            metadata=self.metadata,
+        )
+
+
+class GoldExampleInput(BaseModel):
+    """Input schema for a gold example (contrastive verifier).
+    
+    Validates that the example has required fields with correct types.
+    Uses synth_ai.data.rewards.GoldExample dataclass for structure.
+    """
+    
+    summary: str = Field(..., min_length=1, description="Summary of the trace being evaluated")
+    gold_score: Annotated[float, Field(ge=0.0, le=1.0)] = Field(
+        ...,
+        description="Gold-standard score (0.0-1.0)"
+    )
+    gold_reasoning: str = Field(..., min_length=1, description="Gold-standard reasoning/explanation")
+    session_trace: Optional[dict[str, Any]] = Field(
+        None,
+        description="Optional full trace (for richer evaluation)"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Optional metadata")
+    
+    def to_dataclass(self) -> "GoldExample":
+        """Convert to synth_ai.data.rewards.GoldExample dataclass."""
+        from synth_ai.data.rewards import GoldExample
+        return GoldExample(
+            summary=self.summary,
+            gold_score=self.gold_score,
+            gold_reasoning=self.gold_reasoning,
+            session_trace=self.session_trace,
+            metadata=self.metadata,
+        )

synth_ai/sdk/inference/__init__.py

@@ -0,0 +1,6 @@
+from .client import InferenceClient
+
+__all__ = [
+    "InferenceClient",
+]
+

synth_ai/sdk/inference/client.py

@@ -0,0 +1,128 @@
+"""Inference client for model inference via Synth AI.
+
+This module provides a client for making inference requests through Synth AI's
+inference proxy, which routes requests to appropriate model providers (OpenAI,
+Groq, etc.) based on the model identifier.
+
+Example:
+    >>> from synth_ai.sdk.inference import InferenceClient
+    >>> 
+    >>> client = InferenceClient(
+    ...     base_url="https://api.usesynth.ai",
+    ...     api_key=os.environ["SYNTH_API_KEY"],
+    ... )
+    >>> 
+    >>> response = await client.create_chat_completion(
+    ...     model="gpt-4o-mini",
+    ...     messages=[
+    ...         {"role": "user", "content": "Hello!"}
+    ...     ],
+    ...     temperature=0.7,
+    ...     max_tokens=100,
+    ... )
+    >>> 
+    >>> print(response["choices"][0]["message"]["content"])
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from synth_ai.core._utils.http import AsyncHttpClient
+from synth_ai.sdk.api.models.supported import (
+    UnsupportedModelError,
+    normalize_model_identifier,
+)
+
+
+class InferenceClient:
+    """Client for making inference requests through Synth AI's inference proxy.
+    
+    This client provides a unified interface for calling LLMs through Synth AI's
+    backend, which handles routing to appropriate providers (OpenAI, Groq, etc.)
+    based on the model identifier.
+    
+    Example:
+        >>> from synth_ai.sdk.inference import InferenceClient
+        >>> 
+        >>> client = InferenceClient(
+        ...     base_url="https://api.usesynth.ai",
+        ...     api_key=os.environ["SYNTH_API_KEY"],
+        ... )
+        >>> 
+        >>> # Simple completion
+        >>> response = await client.create_chat_completion(
+        ...     model="gpt-4o-mini",
+        ...     messages=[{"role": "user", "content": "Hello!"}],
+        ... )
+        >>> 
+        >>> # With options
+        >>> response = await client.create_chat_completion(
+        ...     model="Qwen/Qwen3-4B",
+        ...     messages=[{"role": "user", "content": "Explain RL"}],
+        ...     temperature=0.7,
+        ...     max_tokens=500,
+        ...     thinking_budget=512,
+        ... )
+    """
+    
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0) -> None:
+        """Initialize the inference client.
+        
+        Args:
+            base_url: Base URL for the Synth AI API
+            api_key: API key for authentication
+            timeout: Request timeout in seconds (default: 30.0)
+        """
+        self._base_url = base_url.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+
+    async def create_chat_completion(
+        self, *, model: str, messages: list[dict], **kwargs: Any
+    ) -> dict[str, Any]:
+        """Create a chat completion request.
+        
+        This method sends a chat completion request to the Synth AI inference proxy,
+        which routes it to the appropriate provider based on the model identifier.
+        
+        Args:
+            model: Model identifier (e.g., "gpt-4o-mini", "Qwen/Qwen3-4B")
+            messages: List of message dicts with "role" and "content" keys
+            **kwargs: Additional OpenAI-compatible parameters:
+                - temperature: Sampling temperature (0.0-2.0)
+                - max_tokens: Maximum tokens to generate
+                - thinking_budget: Budget for thinking tokens (default: 256)
+                - top_p: Nucleus sampling parameter
+                - frequency_penalty: Frequency penalty (-2.0 to 2.0)
+                - presence_penalty: Presence penalty (-2.0 to 2.0)
+                - stop: Stop sequences
+                - tools: Function calling tools
+                - tool_choice: Tool choice strategy
+                - stream: Whether to stream responses
+                - ... (other OpenAI API parameters)
+                
+        Returns:
+            Completion response dict with:
+                - id: Request ID
+                - choices: List of completion choices
+                - usage: Token usage statistics
+                - ... (other OpenAI-compatible fields)
+                
+        Raises:
+            ValueError: If model is not supported or request is invalid
+            HTTPError: If the API request fails
+        """
+        try:
+            normalized_model = normalize_model_identifier(model)
+        except UnsupportedModelError as exc:
+            raise ValueError(str(exc)) from exc
+
+        body: dict[str, Any] = {"model": normalized_model, "messages": messages}
+        body.update(kwargs)
+        # Backend now expects an explicit thinking_budget; provide a sensible default if omitted
+        if "thinking_budget" not in body:
+            body["thinking_budget"] = 256
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            # Route through backend inference proxy to Modal
+            return await http.post_json("/api/inference/v1/chat/completions", json=body)

synth_ai/sdk/jobs/__init__.py

@@ -0,0 +1,16 @@
+"""Jobs API client for Synth AI."""
+from synth_ai.sdk.jobs.client import (
+    FilesApi,
+    JobsClient,
+    ModelsApi,
+    RlJobsApi,
+    SftJobsApi,
+)
+
+__all__ = [
+    "JobsClient",
+    "FilesApi",
+    "SftJobsApi",
+    "RlJobsApi",
+    "ModelsApi",
+]