synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/judge_schemas.py

@@ -9,7 +9,7 @@ This is the canonical contract that the backend MUST conform to.
 
 from __future__ import annotations
 
-from typing import Any, Literal
+from typing import Any, Literal, Optional
 
 from pydantic import BaseModel, Field
 
@@ -31,7 +31,7 @@ class ReviewPayload(BaseModel):
         description="Map of criterion keys to their scores"
     )
     total: float = Field(default=0.0, description="Aggregated total score")
-    summary: str | None = Field(None, description="Optional text summary")
+    summary: Optional[str] = Field(None, description="Optional text summary")
 
 
 class JudgeScoreResponse(BaseModel):
@@ -46,7 +46,7 @@ class JudgeScoreResponse(BaseModel):
         default_factory=list,
         description="List of per-event rubric reviews (one per step)"
     )
-    outcome_review: ReviewPayload | None = Field(
+    outcome_review: Optional[ReviewPayload] = Field(
         None,
         description="Optional outcome-level rubric review"
     )
@@ -63,7 +63,7 @@ class JudgeScoreResponse(BaseModel):
         description="Request metadata (provider, options, etc.)"
     )
 
-    def aggregate_event_reward(self) -> float | None:
+    def aggregate_event_reward(self) -> Optional[float]:
         """
         Aggregate all event totals into a single reward.
         
@@ -74,7 +74,7 @@ class JudgeScoreResponse(BaseModel):
             return None
         return sum(self.event_totals)
 
-    def aggregate_outcome_reward(self) -> float | None:
+    def aggregate_outcome_reward(self) -> Optional[float]:
         """
         Extract outcome reward from outcome_review.
         
@@ -92,15 +92,15 @@ class JudgeTaskApp(BaseModel):
     """Task application metadata."""
     
     id: str = Field(..., description="Task app identifier")
-    base_url: str | None = Field(None, description="Optional base URL for task app")
+    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
 
 
 class JudgeOptions(BaseModel):
     """Judge provider and configuration options."""
     
-    provider: str | None = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
-    model: str | None = Field(None, description="Model identifier")
-    rubric_id: str | None = Field(None, description="Rubric identifier")
+    provider: Optional[str] = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: Optional[str] = Field(None, description="Model identifier")
+    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
     event: bool = Field(True, description="Enable event-level judging")
     outcome: bool = Field(True, description="Enable outcome-level judging")
 
@@ -123,5 +123,5 @@ class JudgeScoreRequest(BaseModel):
     task_app: JudgeTaskApp = Field(..., description="Task application metadata")
     trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
     options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
-    rubric: dict[str, Any] | None = Field(None, description="Optional explicit rubric criteria")
+    rubric: Optional[dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
 

synth_ai/learning/__init__.py

@@ -3,6 +3,12 @@ from synth_ai.task import task_app_health, validate_task_app_url
 from .client import LearningClient
 from .health import backend_health, balance_autumn_normalized, pricing_preflight
 from .jobs import JobHandle, JobsApiResolver
+from .prompt_learning_client import (
+    PromptLearningClient,
+    get_prompt_text,
+    get_prompts,
+    get_scoring_summary,
+)
 from .rl import (
     MAX_ENVIRONMENT_API_KEY_BYTES,
     RlClient,
@@ -32,6 +38,10 @@ __all__ = [
     "FtClient",
     "SFTJobConfig",
     "prepare_sft_job_payload",
+    "PromptLearningClient",
+    "get_prompts",
+    "get_prompt_text",
+    "get_scoring_summary",
     "RolloutEnvSpec",
     "RolloutPolicySpec",
     "RolloutRecordConfig",

synth_ai/learning/prompt_learning_client.py

@@ -0,0 +1,276 @@
+"""Client utilities for querying prompt learning job results."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from .._utils.http import AsyncHttpClient
+from .prompt_learning_types import PromptResults
+
+
+def _validate_job_id(job_id: str) -> None:
+    """Validate that job_id has the expected prompt learning format.
+    
+    Args:
+        job_id: Job ID to validate
+        
+    Raises:
+        ValueError: If job_id doesn't start with 'pl_'
+    """
+    if not job_id.startswith("pl_"):
+        raise ValueError(
+            f"Invalid prompt learning job ID format: {job_id!r}. "
+            f"Expected format: 'pl_<identifier>' (e.g., 'pl_9c58b711c2644083')"
+        )
+
+
+class PromptLearningClient:
+    """Client for interacting with prompt learning jobs and retrieving results."""
+
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0) -> None:
+        """Initialize the prompt learning client.
+        
+        Args:
+            base_url: Base URL of the backend API (e.g., "http://localhost:8000")
+            api_key: API key for authentication
+            timeout: Request timeout in seconds
+        """
+        self._base_url = base_url.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+
+    async def get_job(self, job_id: str) -> Dict[str, Any]:
+        """Get job metadata and status.
+        
+        Args:
+            job_id: Job ID (e.g., "pl_9c58b711c2644083")
+            
+        Returns:
+            Job metadata including status, best_score, created_at, etc.
+            
+        Raises:
+            ValueError: If job_id format is invalid
+        """
+        _validate_job_id(job_id)
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            return await http.get(f"/api/prompt-learning/online/jobs/{job_id}")
+
+    async def get_events(
+        self, job_id: str, *, since_seq: int = 0, limit: int = 5000
+    ) -> List[Dict[str, Any]]:
+        """Get events for a prompt learning job.
+        
+        Args:
+            job_id: Job ID
+            since_seq: Return events after this sequence number
+            limit: Maximum number of events to return
+            
+        Returns:
+            List of event dictionaries with type, message, data, etc.
+            
+        Raises:
+            ValueError: If job_id format is invalid or response structure is unexpected
+        """
+        _validate_job_id(job_id)
+        params = {"since_seq": since_seq, "limit": limit}
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            js = await http.get(
+                f"/api/prompt-learning/online/jobs/{job_id}/events",
+                params=params
+            )
+        if isinstance(js, dict) and isinstance(js.get("events"), list):
+            return js["events"]
+        # Unexpected response structure - raise instead of silently returning empty list
+        raise ValueError(
+            f"Unexpected response structure from events endpoint. "
+            f"Expected dict with 'events' list, got: {type(js).__name__}"
+        )
+
+    async def get_prompts(self, job_id: str) -> PromptResults:
+        """Get the best prompts and scoring metadata from a completed job.
+        
+        Args:
+            job_id: Job ID
+            
+        Returns:
+            PromptResults dataclass containing:
+                - best_prompt: The top-performing prompt with sections and metadata
+                - best_score: The best accuracy score achieved
+                - top_prompts: List of top-K prompts with train/val scores
+                - optimized_candidates: All frontier/Pareto-optimal candidates
+                - attempted_candidates: All candidates tried during optimization
+                
+        Raises:
+            ValueError: If job_id format is invalid
+        """
+        _validate_job_id(job_id)
+        events = await self.get_events(job_id, limit=10000)
+        
+        result = PromptResults()
+        
+        # Extract results from events
+        for event in events:
+            event_type = event.get("type", "")
+            event_data = event.get("data", {})
+            
+            # Best prompt event
+            if event_type == "prompt.learning.best.prompt":
+                result.best_prompt = event_data.get("best_prompt")
+                result.best_score = event_data.get("best_score")
+            
+            # Top-K prompt content events
+            elif event_type == "prompt.learning.top.prompt.content":
+                result.top_prompts.append({
+                    "rank": event_data.get("rank"),
+                    "train_accuracy": event_data.get("train_accuracy"),
+                    "val_accuracy": event_data.get("val_accuracy"),
+                    "template": event_data.get("template"),
+                    "full_text": event_data.get("full_text"),
+                })
+            
+            # Final results event (contains all candidates)
+            elif event_type == "prompt.learning.final.results":
+                result.optimized_candidates = event_data.get("optimized_candidates", [])
+                result.attempted_candidates = event_data.get("attempted_candidates", [])
+            
+            # Validation results
+            elif event_type == "prompt.learning.validation.scored":
+                result.validation_results.append(event_data)
+            
+            # Completion event (fallback for best_score)
+            elif event_type == "prompt.learning.gepa.complete":
+                if result.best_score is None:
+                    result.best_score = event_data.get("best_score")
+        
+        return result
+
+    async def get_prompt_text(self, job_id: str, rank: int = 1) -> Optional[str]:
+        """Get the full text of a specific prompt by rank.
+        
+        Args:
+            job_id: Job ID
+            rank: Prompt rank (1 = best, 2 = second best, etc.)
+            
+        Returns:
+            Full prompt text or None if not found
+            
+        Raises:
+            ValueError: If job_id format is invalid or rank < 1
+        """
+        _validate_job_id(job_id)
+        if rank < 1:
+            raise ValueError(f"Rank must be >= 1, got: {rank}")
+        prompts_data = await self.get_prompts(job_id)
+        top_prompts = prompts_data.top_prompts
+        
+        for prompt_info in top_prompts:
+            if prompt_info.get("rank") == rank:
+                return prompt_info.get("full_text")
+        
+        return None
+
+    async def get_scoring_summary(self, job_id: str) -> Dict[str, Any]:
+        """Get a summary of scoring metrics for all candidates.
+        
+        Args:
+            job_id: Job ID
+            
+        Returns:
+            Dictionary with scoring statistics:
+                - best_train_accuracy: Best training accuracy
+                - best_val_accuracy: Best validation accuracy (if available)
+                - num_candidates_tried: Total candidates evaluated
+                - num_frontier_candidates: Number in Pareto frontier
+                - score_distribution: Histogram of accuracy scores
+                
+        Raises:
+            ValueError: If job_id format is invalid
+        """
+        _validate_job_id(job_id)
+        prompts_data = await self.get_prompts(job_id)
+        
+        attempted = prompts_data.attempted_candidates
+        optimized = prompts_data.optimized_candidates
+        validation = prompts_data.validation_results
+        
+        # Extract train accuracies (only from candidates that have accuracy field)
+        train_accuracies = [
+            c["accuracy"] for c in attempted if "accuracy" in c
+        ]
+        
+        # Extract val accuracies (only from validations that have accuracy field)
+        val_accuracies = [
+            v["accuracy"] for v in validation if "accuracy" in v
+        ]
+        
+        # Score distribution (bins)
+        bins = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
+        distribution = {f"{bins[i]:.1f}-{bins[i+1]:.1f}": 0 for i in range(len(bins) - 1)}
+        for acc in train_accuracies:
+            for i in range(len(bins) - 1):
+                if bins[i] <= acc < bins[i+1] or (i == len(bins) - 2 and acc == bins[i+1]):
+                    distribution[f"{bins[i]:.1f}-{bins[i+1]:.1f}"] += 1
+                    break
+        
+        return {
+            "best_train_accuracy": max(train_accuracies) if train_accuracies else None,
+            "best_val_accuracy": max(val_accuracies) if val_accuracies else None,
+            "num_candidates_tried": len(attempted),
+            "num_frontier_candidates": len(optimized),
+            "score_distribution": distribution,
+            "mean_train_accuracy": sum(train_accuracies) / len(train_accuracies) if train_accuracies else None,
+        }
+
+
+# Synchronous wrapper for convenience
+def get_prompts(job_id: str, base_url: str, api_key: str) -> PromptResults:
+    """Synchronous wrapper to get prompts from a job.
+    
+    Args:
+        job_id: Job ID (e.g., "pl_9c58b711c2644083")
+        base_url: Backend API base URL
+        api_key: API key for authentication
+        
+    Returns:
+        PromptResults dataclass with prompt results
+    """
+    import asyncio
+    
+    client = PromptLearningClient(base_url, api_key)
+    return asyncio.run(client.get_prompts(job_id))
+
+
+def get_prompt_text(job_id: str, base_url: str, api_key: str, rank: int = 1) -> Optional[str]:
+    """Synchronous wrapper to get prompt text by rank.
+    
+    Args:
+        job_id: Job ID
+        base_url: Backend API base URL
+        api_key: API key for authentication
+        rank: Prompt rank (1 = best, 2 = second best, etc.)
+        
+    Returns:
+        Full prompt text or None if not found
+    """
+    import asyncio
+    
+    client = PromptLearningClient(base_url, api_key)
+    return asyncio.run(client.get_prompt_text(job_id, rank))
+
+
+def get_scoring_summary(job_id: str, base_url: str, api_key: str) -> Dict[str, Any]:
+    """Synchronous wrapper to get scoring summary.
+    
+    Args:
+        job_id: Job ID
+        base_url: Backend API base URL
+        api_key: API key for authentication
+        
+    Returns:
+        Dictionary with scoring statistics
+    """
+    import asyncio
+    
+    client = PromptLearningClient(base_url, api_key)
+    return asyncio.run(client.get_scoring_summary(job_id))
+

synth_ai/learning/prompt_learning_types.py

@@ -0,0 +1,184 @@
+"""Type definitions for prompt learning data structures."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class TextReplacement:
+    """A text replacement in a prompt transformation."""
+    
+    new_text: str
+    apply_to_role: str = "system"
+    old_text: Optional[str] = None
+    position: Optional[int] = None
+
+
+@dataclass
+class CandidateScore:
+    """Scoring information for a candidate prompt."""
+    
+    accuracy: float
+    prompt_length: int = 0
+    tool_call_rate: float = 0.0
+    instance_scores: List[float] = field(default_factory=list)
+
+
+@dataclass
+class PromptSection:
+    """A section of a prompt (e.g., system, user, assistant)."""
+    
+    role: str
+    content: str
+
+
+@dataclass
+class Candidate:
+    """A candidate prompt from the optimization process."""
+    
+    accuracy: float
+    prompt_length: int = 0
+    tool_call_rate: float = 0.0
+    instance_scores: List[float] = field(default_factory=list)
+    object: Optional[Dict[str, Any]] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> Candidate:
+        """Create a Candidate from a dictionary."""
+        return cls(
+            accuracy=data.get("accuracy", 0.0),
+            prompt_length=data.get("prompt_length", 0),
+            tool_call_rate=data.get("tool_call_rate", 0.0),
+            instance_scores=data.get("instance_scores", []),
+            object=data.get("object"),
+        )
+
+
+@dataclass
+class OptimizedCandidate:
+    """An optimized candidate from the Pareto frontier."""
+    
+    score: CandidateScore
+    payload_kind: str  # "transformation" or "template"
+    object: Optional[Dict[str, Any]] = None
+    instance_scores: Optional[List[float]] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> OptimizedCandidate:
+        """Create an OptimizedCandidate from a dictionary."""
+        score_data = data.get("score", {})
+        if isinstance(score_data, dict):
+            score = CandidateScore(
+                accuracy=score_data.get("accuracy", 0.0),
+                prompt_length=score_data.get("prompt_length", 0),
+                tool_call_rate=score_data.get("tool_call_rate", 0.0),
+                instance_scores=score_data.get("instance_scores", []),
+            )
+        else:
+            score = CandidateScore(accuracy=0.0)
+        
+        return cls(
+            score=score,
+            payload_kind=data.get("payload_kind", "unknown"),
+            object=data.get("object"),
+            instance_scores=data.get("instance_scores"),
+        )
+
+
+@dataclass
+class PromptLearningEvent:
+    """A generic prompt learning event."""
+    
+    type: str
+    message: str
+    data: Dict[str, Any]
+    seq: int
+    created_at: Optional[str] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> PromptLearningEvent:
+        """Create a PromptLearningEvent from a dictionary."""
+        return cls(
+            type=data.get("type", ""),
+            message=data.get("message", ""),
+            data=data.get("data", {}),
+            seq=data.get("seq", 0),
+            created_at=data.get("created_at"),
+        )
+
+
+@dataclass
+class BestPromptEventData:
+    """Data for prompt.learning.best.prompt event."""
+    
+    best_score: float
+    best_prompt: Dict[str, Any]
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> BestPromptEventData:
+        """Create BestPromptEventData from a dictionary."""
+        return cls(
+            best_score=data.get("best_score", 0.0),
+            best_prompt=data.get("best_prompt", {}),
+        )
+
+
+@dataclass
+class FinalResultsEventData:
+    """Data for prompt.learning.final.results event."""
+    
+    attempted_candidates: List[Dict[str, Any]]
+    optimized_candidates: List[Dict[str, Any]]
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> FinalResultsEventData:
+        """Create FinalResultsEventData from a dictionary."""
+        return cls(
+            attempted_candidates=data.get("attempted_candidates", []),
+            optimized_candidates=data.get("optimized_candidates", []),
+        )
+
+
+@dataclass
+class ValidationScoredEventData:
+    """Data for prompt.learning.validation.scored event."""
+    
+    accuracy: float
+    instance_scores: List[float] = field(default_factory=list)
+    is_baseline: bool = False
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> ValidationScoredEventData:
+        """Create ValidationScoredEventData from a dictionary."""
+        return cls(
+            accuracy=data.get("accuracy", 0.0),
+            instance_scores=data.get("instance_scores", []),
+            is_baseline=data.get("is_baseline", False),
+        )
+
+
+@dataclass
+class PromptResults:
+    """Results from a completed prompt learning job."""
+    
+    best_prompt: Optional[Dict[str, Any]] = None
+    best_score: Optional[float] = None
+    top_prompts: List[Dict[str, Any]] = field(default_factory=list)
+    optimized_candidates: List[Dict[str, Any]] = field(default_factory=list)
+    attempted_candidates: List[Dict[str, Any]] = field(default_factory=list)
+    validation_results: List[Dict[str, Any]] = field(default_factory=list)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> PromptResults:
+        """Create PromptResults from a dictionary."""
+        return cls(
+            best_prompt=data.get("best_prompt"),
+            best_score=data.get("best_score"),
+            top_prompts=data.get("top_prompts", []),
+            optimized_candidates=data.get("optimized_candidates", []),
+            attempted_candidates=data.get("attempted_candidates", []),
+            validation_results=data.get("validation_results", []),
+        )
+

synth_ai/learning/rl/client.py

@@ -107,7 +107,9 @@ class RlClient:
         async with AsyncHttpClient(self._base_url, self._api_key, timeout=30.0) as http:
             try:
                 js = await http.get(
-                    f"{_api_base(self._base_url)}/learning/jobs/{job_id}/events", params=params
+                    f"{_api_base(self._base_url)}/learning/jobs/{job_id}/events",
+                    params=params,
+                    headers={"accept": "application/json"},
                 )
             except HTTPError as he:
                 with suppress(Exception):

synth_ai/pricing/__init__.py

@@ -0,0 +1,2 @@
+"""Pricing module for SDK."""
+

synth_ai/pricing/model_pricing.py

@@ -0,0 +1,57 @@
+"""Static pricing table for supported models.
+
+This module provides per-token pricing used by the SDK status commands.
+Rates are expressed in USD per token and split into input/output prices.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class TokenRates:
+    input_usd: float
+    output_usd: float
+
+
+# Default per-token prices (USD), sourced Nov 3, 2025 — update as contracts change
+MODEL_PRICES: Dict[str, Dict[str, TokenRates]] = {
+    # OpenAI official pricing
+    "openai": {
+        # GPT-5 family
+        "gpt-5":       TokenRates(input_usd=0.00000125, output_usd=0.00001000),  # $1.25 / $10 per 1M
+        "gpt-5-mini":  TokenRates(input_usd=0.00000025, output_usd=0.00000200),  # $0.25 / $2.00 per 1M
+        "gpt-5-nano":  TokenRates(input_usd=0.00000005, output_usd=0.00000040),  # $0.05 / $0.40 per 1M
+
+        "gpt-4o-mini": TokenRates(input_usd=0.00000015, output_usd=0.00000060),  # $0.15 / $0.60 per 1M
+        "gpt-4o":      TokenRates(input_usd=0.00000250, output_usd=0.00001000),  # $2.50 / $10.00 per 1M
+    },
+    # Groq OSS via OpenAI-compatible path (latest Groq docs)
+    "groq": {
+        "openai/gpt-oss-20b":  TokenRates(input_usd=0.000000075, output_usd=0.000000300),  # $0.075 / $0.30 per 1M
+        
+        "openai/gpt-oss-120b": TokenRates(input_usd=0.000000150, output_usd=0.000000600),  # $0.15 / $0.60 per 1M
+
+        # Additional Groq on-demand models
+        "moonshotai/kimi-k2-0905":        TokenRates(input_usd=0.000001000, output_usd=0.000003000),  # $1.00 / $3.00 per 1M
+        
+        "meta/llama-guard-4-12b":         TokenRates(input_usd=0.000000200, output_usd=0.000000200),  # $0.20 / $0.20 per 1M
+        "qwen/qwen3-32b":                 TokenRates(input_usd=0.000000290, output_usd=0.000000590),  # $0.29 / $0.59 per 1M
+        "meta/llama-3.3-70b-versatile":   TokenRates(input_usd=0.000000590, output_usd=0.000000790),  # $0.59 / $0.79 per 1M
+        "meta/llama-3.1-8b-instant":      TokenRates(input_usd=0.000000050, output_usd=0.000000080),  # $0.05 / $0.08 per 1M
+    },
+    # Google Gemini pricing — per-token USD (per 1M ÷ 1e6), Nov 3, 2025
+    "google": {
+        # Gemini 2.5 Pro (two tiers by prompt size)
+        "gemini-2.5-pro":        TokenRates(input_usd=0.00000125, output_usd=0.00001000),  # <=200k tokens
+        "gemini-2.5-pro-gt200k": TokenRates(input_usd=0.00000250, output_usd=0.00001500),  # >200k tokens
+
+        # Gemini 2.5 Flash (hybrid reasoning)
+        "gemini-2.5-flash":      TokenRates(input_usd=0.00000030, output_usd=0.00000250),
+
+        # Gemini 2.5 Flash-Lite (cheapest)
+        "gemini-2.5-flash-lite": TokenRates(input_usd=0.00000010, output_usd=0.00000040),
+    },
+}
+

synth_ai/streaming/__init__.py

@@ -0,0 +1,29 @@
+from .config import StreamConfig
+from .handlers import (
+    BufferedHandler,
+    CallbackHandler,
+    CLIHandler,
+    IntegrationTestHandler,
+    JSONHandler,
+    LossCurveHandler,
+    RichHandler,
+    StreamHandler,
+)
+from .streamer import JobStreamer, StreamEndpoints
+from .types import StreamMessage, StreamType
+
+__all__ = [
+    "BufferedHandler",
+    "CallbackHandler",
+    "CLIHandler",
+    "IntegrationTestHandler",
+    "JSONHandler",
+    "LossCurveHandler",
+    "JobStreamer",
+    "RichHandler",
+    "StreamEndpoints",
+    "StreamConfig",
+    "StreamHandler",
+    "StreamMessage",
+    "StreamType",
+]