synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/learning/prompt_learning_client.py

@@ -0,0 +1,276 @@
+"""Client utilities for querying prompt learning job results."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from .._utils.http import AsyncHttpClient
+from .prompt_learning_types import PromptResults
+
+
+def _validate_job_id(job_id: str) -> None:
+    """Validate that job_id has the expected prompt learning format.
+    
+    Args:
+        job_id: Job ID to validate
+        
+    Raises:
+        ValueError: If job_id doesn't start with 'pl_'
+    """
+    if not job_id.startswith("pl_"):
+        raise ValueError(
+            f"Invalid prompt learning job ID format: {job_id!r}. "
+            f"Expected format: 'pl_<identifier>' (e.g., 'pl_9c58b711c2644083')"
+        )
+
+
+class PromptLearningClient:
+    """Client for interacting with prompt learning jobs and retrieving results."""
+
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0) -> None:
+        """Initialize the prompt learning client.
+        
+        Args:
+            base_url: Base URL of the backend API (e.g., "http://localhost:8000")
+            api_key: API key for authentication
+            timeout: Request timeout in seconds
+        """
+        self._base_url = base_url.rstrip("/")
+        self._api_key = api_key
+        self._timeout = timeout
+
+    async def get_job(self, job_id: str) -> Dict[str, Any]:
+        """Get job metadata and status.
+        
+        Args:
+            job_id: Job ID (e.g., "pl_9c58b711c2644083")
+            
+        Returns:
+            Job metadata including status, best_score, created_at, etc.
+            
+        Raises:
+            ValueError: If job_id format is invalid
+        """
+        _validate_job_id(job_id)
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            return await http.get(f"/api/prompt-learning/online/jobs/{job_id}")
+
+    async def get_events(
+        self, job_id: str, *, since_seq: int = 0, limit: int = 5000
+    ) -> List[Dict[str, Any]]:
+        """Get events for a prompt learning job.
+        
+        Args:
+            job_id: Job ID
+            since_seq: Return events after this sequence number
+            limit: Maximum number of events to return
+            
+        Returns:
+            List of event dictionaries with type, message, data, etc.
+            
+        Raises:
+            ValueError: If job_id format is invalid or response structure is unexpected
+        """
+        _validate_job_id(job_id)
+        params = {"since_seq": since_seq, "limit": limit}
+        async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
+            js = await http.get(
+                f"/api/prompt-learning/online/jobs/{job_id}/events",
+                params=params
+            )
+        if isinstance(js, dict) and isinstance(js.get("events"), list):
+            return js["events"]
+        # Unexpected response structure - raise instead of silently returning empty list
+        raise ValueError(
+            f"Unexpected response structure from events endpoint. "
+            f"Expected dict with 'events' list, got: {type(js).__name__}"
+        )
+
+    async def get_prompts(self, job_id: str) -> PromptResults:
+        """Get the best prompts and scoring metadata from a completed job.
+        
+        Args:
+            job_id: Job ID
+            
+        Returns:
+            PromptResults dataclass containing:
+                - best_prompt: The top-performing prompt with sections and metadata
+                - best_score: The best accuracy score achieved
+                - top_prompts: List of top-K prompts with train/val scores
+                - optimized_candidates: All frontier/Pareto-optimal candidates
+                - attempted_candidates: All candidates tried during optimization
+                
+        Raises:
+            ValueError: If job_id format is invalid
+        """
+        _validate_job_id(job_id)
+        events = await self.get_events(job_id, limit=10000)
+        
+        result = PromptResults()
+        
+        # Extract results from events
+        for event in events:
+            event_type = event.get("type", "")
+            event_data = event.get("data", {})
+            
+            # Best prompt event
+            if event_type == "prompt.learning.best.prompt":
+                result.best_prompt = event_data.get("best_prompt")
+                result.best_score = event_data.get("best_score")
+            
+            # Top-K prompt content events
+            elif event_type == "prompt.learning.top.prompt.content":
+                result.top_prompts.append({
+                    "rank": event_data.get("rank"),
+                    "train_accuracy": event_data.get("train_accuracy"),
+                    "val_accuracy": event_data.get("val_accuracy"),
+                    "template": event_data.get("template"),
+                    "full_text": event_data.get("full_text"),
+                })
+            
+            # Final results event (contains all candidates)
+            elif event_type == "prompt.learning.final.results":
+                result.optimized_candidates = event_data.get("optimized_candidates", [])
+                result.attempted_candidates = event_data.get("attempted_candidates", [])
+            
+            # Validation results
+            elif event_type == "prompt.learning.validation.scored":
+                result.validation_results.append(event_data)
+            
+            # Completion event (fallback for best_score)
+            elif event_type == "prompt.learning.gepa.complete":
+                if result.best_score is None:
+                    result.best_score = event_data.get("best_score")
+        
+        return result
+
+    async def get_prompt_text(self, job_id: str, rank: int = 1) -> Optional[str]:
+        """Get the full text of a specific prompt by rank.
+        
+        Args:
+            job_id: Job ID
+            rank: Prompt rank (1 = best, 2 = second best, etc.)
+            
+        Returns:
+            Full prompt text or None if not found
+            
+        Raises:
+            ValueError: If job_id format is invalid or rank < 1
+        """
+        _validate_job_id(job_id)
+        if rank < 1:
+            raise ValueError(f"Rank must be >= 1, got: {rank}")
+        prompts_data = await self.get_prompts(job_id)
+        top_prompts = prompts_data.top_prompts
+        
+        for prompt_info in top_prompts:
+            if prompt_info.get("rank") == rank:
+                return prompt_info.get("full_text")
+        
+        return None
+
+    async def get_scoring_summary(self, job_id: str) -> Dict[str, Any]:
+        """Get a summary of scoring metrics for all candidates.
+        
+        Args:
+            job_id: Job ID
+            
+        Returns:
+            Dictionary with scoring statistics:
+                - best_train_accuracy: Best training accuracy
+                - best_val_accuracy: Best validation accuracy (if available)
+                - num_candidates_tried: Total candidates evaluated
+                - num_frontier_candidates: Number in Pareto frontier
+                - score_distribution: Histogram of accuracy scores
+                
+        Raises:
+            ValueError: If job_id format is invalid
+        """
+        _validate_job_id(job_id)
+        prompts_data = await self.get_prompts(job_id)
+        
+        attempted = prompts_data.attempted_candidates
+        optimized = prompts_data.optimized_candidates
+        validation = prompts_data.validation_results
+        
+        # Extract train accuracies (only from candidates that have accuracy field)
+        train_accuracies = [
+            c["accuracy"] for c in attempted if "accuracy" in c
+        ]
+        
+        # Extract val accuracies (only from validations that have accuracy field)
+        val_accuracies = [
+            v["accuracy"] for v in validation if "accuracy" in v
+        ]
+        
+        # Score distribution (bins)
+        bins = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
+        distribution = {f"{bins[i]:.1f}-{bins[i+1]:.1f}": 0 for i in range(len(bins) - 1)}
+        for acc in train_accuracies:
+            for i in range(len(bins) - 1):
+                if bins[i] <= acc < bins[i+1] or (i == len(bins) - 2 and acc == bins[i+1]):
+                    distribution[f"{bins[i]:.1f}-{bins[i+1]:.1f}"] += 1
+                    break
+        
+        return {
+            "best_train_accuracy": max(train_accuracies) if train_accuracies else None,
+            "best_val_accuracy": max(val_accuracies) if val_accuracies else None,
+            "num_candidates_tried": len(attempted),
+            "num_frontier_candidates": len(optimized),
+            "score_distribution": distribution,
+            "mean_train_accuracy": sum(train_accuracies) / len(train_accuracies) if train_accuracies else None,
+        }
+
+
+# Synchronous wrapper for convenience
+def get_prompts(job_id: str, base_url: str, api_key: str) -> PromptResults:
+    """Synchronous wrapper to get prompts from a job.
+    
+    Args:
+        job_id: Job ID (e.g., "pl_9c58b711c2644083")
+        base_url: Backend API base URL
+        api_key: API key for authentication
+        
+    Returns:
+        PromptResults dataclass with prompt results
+    """
+    import asyncio
+    
+    client = PromptLearningClient(base_url, api_key)
+    return asyncio.run(client.get_prompts(job_id))
+
+
+def get_prompt_text(job_id: str, base_url: str, api_key: str, rank: int = 1) -> Optional[str]:
+    """Synchronous wrapper to get prompt text by rank.
+    
+    Args:
+        job_id: Job ID
+        base_url: Backend API base URL
+        api_key: API key for authentication
+        rank: Prompt rank (1 = best, 2 = second best, etc.)
+        
+    Returns:
+        Full prompt text or None if not found
+    """
+    import asyncio
+    
+    client = PromptLearningClient(base_url, api_key)
+    return asyncio.run(client.get_prompt_text(job_id, rank))
+
+
+def get_scoring_summary(job_id: str, base_url: str, api_key: str) -> Dict[str, Any]:
+    """Synchronous wrapper to get scoring summary.
+    
+    Args:
+        job_id: Job ID
+        base_url: Backend API base URL
+        api_key: API key for authentication
+        
+    Returns:
+        Dictionary with scoring statistics
+    """
+    import asyncio
+    
+    client = PromptLearningClient(base_url, api_key)
+    return asyncio.run(client.get_scoring_summary(job_id))
+

synth_ai/learning/prompt_learning_types.py

@@ -0,0 +1,184 @@
+"""Type definitions for prompt learning data structures."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class TextReplacement:
+    """A text replacement in a prompt transformation."""
+    
+    new_text: str
+    apply_to_role: str = "system"
+    old_text: Optional[str] = None
+    position: Optional[int] = None
+
+
+@dataclass
+class CandidateScore:
+    """Scoring information for a candidate prompt."""
+    
+    accuracy: float
+    prompt_length: int = 0
+    tool_call_rate: float = 0.0
+    instance_scores: List[float] = field(default_factory=list)
+
+
+@dataclass
+class PromptSection:
+    """A section of a prompt (e.g., system, user, assistant)."""
+    
+    role: str
+    content: str
+
+
+@dataclass
+class Candidate:
+    """A candidate prompt from the optimization process."""
+    
+    accuracy: float
+    prompt_length: int = 0
+    tool_call_rate: float = 0.0
+    instance_scores: List[float] = field(default_factory=list)
+    object: Optional[Dict[str, Any]] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> Candidate:
+        """Create a Candidate from a dictionary."""
+        return cls(
+            accuracy=data.get("accuracy", 0.0),
+            prompt_length=data.get("prompt_length", 0),
+            tool_call_rate=data.get("tool_call_rate", 0.0),
+            instance_scores=data.get("instance_scores", []),
+            object=data.get("object"),
+        )
+
+
+@dataclass
+class OptimizedCandidate:
+    """An optimized candidate from the Pareto frontier."""
+    
+    score: CandidateScore
+    payload_kind: str  # "transformation" or "template"
+    object: Optional[Dict[str, Any]] = None
+    instance_scores: Optional[List[float]] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> OptimizedCandidate:
+        """Create an OptimizedCandidate from a dictionary."""
+        score_data = data.get("score", {})
+        if isinstance(score_data, dict):
+            score = CandidateScore(
+                accuracy=score_data.get("accuracy", 0.0),
+                prompt_length=score_data.get("prompt_length", 0),
+                tool_call_rate=score_data.get("tool_call_rate", 0.0),
+                instance_scores=score_data.get("instance_scores", []),
+            )
+        else:
+            score = CandidateScore(accuracy=0.0)
+        
+        return cls(
+            score=score,
+            payload_kind=data.get("payload_kind", "unknown"),
+            object=data.get("object"),
+            instance_scores=data.get("instance_scores"),
+        )
+
+
+@dataclass
+class PromptLearningEvent:
+    """A generic prompt learning event."""
+    
+    type: str
+    message: str
+    data: Dict[str, Any]
+    seq: int
+    created_at: Optional[str] = None
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> PromptLearningEvent:
+        """Create a PromptLearningEvent from a dictionary."""
+        return cls(
+            type=data.get("type", ""),
+            message=data.get("message", ""),
+            data=data.get("data", {}),
+            seq=data.get("seq", 0),
+            created_at=data.get("created_at"),
+        )
+
+
+@dataclass
+class BestPromptEventData:
+    """Data for prompt.learning.best.prompt event."""
+    
+    best_score: float
+    best_prompt: Dict[str, Any]
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> BestPromptEventData:
+        """Create BestPromptEventData from a dictionary."""
+        return cls(
+            best_score=data.get("best_score", 0.0),
+            best_prompt=data.get("best_prompt", {}),
+        )
+
+
+@dataclass
+class FinalResultsEventData:
+    """Data for prompt.learning.final.results event."""
+    
+    attempted_candidates: List[Dict[str, Any]]
+    optimized_candidates: List[Dict[str, Any]]
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> FinalResultsEventData:
+        """Create FinalResultsEventData from a dictionary."""
+        return cls(
+            attempted_candidates=data.get("attempted_candidates", []),
+            optimized_candidates=data.get("optimized_candidates", []),
+        )
+
+
+@dataclass
+class ValidationScoredEventData:
+    """Data for prompt.learning.validation.scored event."""
+    
+    accuracy: float
+    instance_scores: List[float] = field(default_factory=list)
+    is_baseline: bool = False
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> ValidationScoredEventData:
+        """Create ValidationScoredEventData from a dictionary."""
+        return cls(
+            accuracy=data.get("accuracy", 0.0),
+            instance_scores=data.get("instance_scores", []),
+            is_baseline=data.get("is_baseline", False),
+        )
+
+
+@dataclass
+class PromptResults:
+    """Results from a completed prompt learning job."""
+    
+    best_prompt: Optional[Dict[str, Any]] = None
+    best_score: Optional[float] = None
+    top_prompts: List[Dict[str, Any]] = field(default_factory=list)
+    optimized_candidates: List[Dict[str, Any]] = field(default_factory=list)
+    attempted_candidates: List[Dict[str, Any]] = field(default_factory=list)
+    validation_results: List[Dict[str, Any]] = field(default_factory=list)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> PromptResults:
+        """Create PromptResults from a dictionary."""
+        return cls(
+            best_prompt=data.get("best_prompt"),
+            best_score=data.get("best_score"),
+            top_prompts=data.get("top_prompts", []),
+            optimized_candidates=data.get("optimized_candidates", []),
+            attempted_candidates=data.get("attempted_candidates", []),
+            validation_results=data.get("validation_results", []),
+        )
+

synth_ai/pricing/__init__.py

@@ -0,0 +1,2 @@
+"""Pricing module for SDK."""
+

synth_ai/pricing/model_pricing.py

@@ -0,0 +1,57 @@
+"""Static pricing table for supported models.
+
+This module provides per-token pricing used by the SDK status commands.
+Rates are expressed in USD per token and split into input/output prices.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class TokenRates:
+    input_usd: float
+    output_usd: float
+
+
+# Default per-token prices (USD), sourced Nov 3, 2025 — update as contracts change
+MODEL_PRICES: Dict[str, Dict[str, TokenRates]] = {
+    # OpenAI official pricing
+    "openai": {
+        # GPT-5 family
+        "gpt-5":       TokenRates(input_usd=0.00000125, output_usd=0.00001000),  # $1.25 / $10 per 1M
+        "gpt-5-mini":  TokenRates(input_usd=0.00000025, output_usd=0.00000200),  # $0.25 / $2.00 per 1M
+        "gpt-5-nano":  TokenRates(input_usd=0.00000005, output_usd=0.00000040),  # $0.05 / $0.40 per 1M
+
+        "gpt-4o-mini": TokenRates(input_usd=0.00000015, output_usd=0.00000060),  # $0.15 / $0.60 per 1M
+        "gpt-4o":      TokenRates(input_usd=0.00000250, output_usd=0.00001000),  # $2.50 / $10.00 per 1M
+    },
+    # Groq OSS via OpenAI-compatible path (latest Groq docs)
+    "groq": {
+        "openai/gpt-oss-20b":  TokenRates(input_usd=0.000000075, output_usd=0.000000300),  # $0.075 / $0.30 per 1M
+        
+        "openai/gpt-oss-120b": TokenRates(input_usd=0.000000150, output_usd=0.000000600),  # $0.15 / $0.60 per 1M
+
+        # Additional Groq on-demand models
+        "moonshotai/kimi-k2-0905":        TokenRates(input_usd=0.000001000, output_usd=0.000003000),  # $1.00 / $3.00 per 1M
+        
+        "meta/llama-guard-4-12b":         TokenRates(input_usd=0.000000200, output_usd=0.000000200),  # $0.20 / $0.20 per 1M
+        "qwen/qwen3-32b":                 TokenRates(input_usd=0.000000290, output_usd=0.000000590),  # $0.29 / $0.59 per 1M
+        "meta/llama-3.3-70b-versatile":   TokenRates(input_usd=0.000000590, output_usd=0.000000790),  # $0.59 / $0.79 per 1M
+        "meta/llama-3.1-8b-instant":      TokenRates(input_usd=0.000000050, output_usd=0.000000080),  # $0.05 / $0.08 per 1M
+    },
+    # Google Gemini pricing — per-token USD (per 1M ÷ 1e6), Nov 3, 2025
+    "google": {
+        # Gemini 2.5 Pro (two tiers by prompt size)
+        "gemini-2.5-pro":        TokenRates(input_usd=0.00000125, output_usd=0.00001000),  # <=200k tokens
+        "gemini-2.5-pro-gt200k": TokenRates(input_usd=0.00000250, output_usd=0.00001500),  # >200k tokens
+
+        # Gemini 2.5 Flash (hybrid reasoning)
+        "gemini-2.5-flash":      TokenRates(input_usd=0.00000030, output_usd=0.00000250),
+
+        # Gemini 2.5 Flash-Lite (cheapest)
+        "gemini-2.5-flash-lite": TokenRates(input_usd=0.00000010, output_usd=0.00000040),
+    },
+}
+

synth_ai/streaming/handlers.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import contextlib
 import json
+import re
 import time
 from abc import ABC, abstractmethod
 from collections import deque
@@ -14,6 +15,37 @@ import click
 from .types import StreamMessage, StreamType
 
 
+def _mask_sensitive_urls(text: str) -> str:
+    """Mask S3/Wasabi URLs and sensitive paths in log messages.
+    
+    Replaces full S3/Wasabi URLs with masked versions to prevent leaking
+    bucket names, paths, and infrastructure details in public SDK logs.
+    
+    Examples:
+        s3://synth-artifacts/models/... -> s3://***/***/[masked]
+        Wasabi s3://bucket/path/file.tar.gz -> Wasabi s3://***/***/[masked]
+    """
+    if not text:
+        return text
+    
+    # Pattern matches:
+    # - Optional "Wasabi " prefix
+    # - s3:// or http(s):// scheme
+    # - Any bucket/host
+    # - Any path
+    # - Common model file extensions
+    pattern = r'(Wasabi\s+)?((s3|https?)://[^\s]+\.(tar\.gz|zip|pt|pth|safetensors|ckpt|bin))'
+    
+    def replace_url(match: re.Match) -> str:
+        prefix = match.group(1) or ""  # "Wasabi " or empty
+        url = match.group(2)
+        # Extract just the filename
+        filename = url.split("/")[-1] if "/" in url else "file"
+        return f'{prefix}s3://***/***/[{filename}]'
+    
+    return re.sub(pattern, replace_url, text, flags=re.IGNORECASE)
+
+
 class StreamHandler(ABC):
     """Base class for log handlers that consume ``StreamMessage`` objects."""
 
@@ -72,14 +104,29 @@ class CLIHandler(StreamHandler):
             prefix = f"[{timestamp}] [{message.seq}] {event_type}"
             if level:
                 prefix += f" ({level})"
-            click.echo(f"{prefix}: {msg}".rstrip(": "))
+            # Mask sensitive URLs before displaying
+            sanitized_msg = _mask_sensitive_urls(msg)
+            click.echo(f"{prefix}: {sanitized_msg}".rstrip(": "))
             return
 
         if message.stream_type is StreamType.METRICS:
-            name = message.data.get("name", "metric")
+            name = message.data.get("name")
             value = message.data.get("value")
             step = message.data.get("step")
-            click.echo(f"[{timestamp}] {name}={value} (step={step})")
+            data = message.data.get("data", {})
+            
+            # Format metric display
+            metric_str = f"[{timestamp}] [metric] {name}={value:.4f}" if isinstance(value, (int, float)) else f"[{timestamp}] [metric] {name}={value}"
+            if step is not None:
+                metric_str += f" (step={step})"
+            
+            # Add any additional context from data field
+            if isinstance(data, dict):
+                n = data.get("n")
+                if n is not None:
+                    metric_str += f" n={n}"
+            
+            click.echo(metric_str)
             return
 
         if message.stream_type is StreamType.TIMELINE:
@@ -387,7 +434,9 @@ class RichHandler(StreamHandler):
             event_type = message.data.get("type", "event")
             summary = message.data.get("message") or ""
             level = message.data.get("level")
-            formatted = f"[{event_type}] {summary}".strip()
+            # Mask sensitive URLs before displaying
+            sanitized_summary = _mask_sensitive_urls(summary)
+            formatted = f"[{event_type}] {sanitized_summary}".strip()
             if level:
                 formatted = f"{formatted} ({level})"
             self._event_log.append(formatted)

synth_ai/streaming/streamer.py

@@ -51,6 +51,25 @@ class StreamEndpoints:
             timeline=f"{base}/timeline",
         )
 
+    @classmethod
+    def prompt_learning(cls, job_id: str) -> StreamEndpoints:
+        """Endpoints for prompt learning jobs (MIPRO/GEPA)."""
+        base = f"/prompt-learning/online/jobs/{job_id}"
+        return cls(
+            status=base,
+            events=f"{base}/events",
+            metrics=f"{base}/metrics",
+            timeline=None,
+            status_fallbacks=(
+                f"/learning/jobs/{job_id}",
+                f"/orchestration/jobs/{job_id}",
+            ),
+            event_fallbacks=(
+                f"/learning/jobs/{job_id}/events",
+                f"/orchestration/jobs/{job_id}/events",
+            ),
+        )
+
     @classmethod
     def rl(cls, job_id: str) -> StreamEndpoints:
         base = f"/rl/jobs/{job_id}"

synth_ai/task/apps/__init__.py

@@ -22,6 +22,7 @@ class ModalDeploymentConfig:
     extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
     secret_names: Sequence[str] = field(default_factory=tuple)
     volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    env_vars: dict[str, str] = field(default_factory=dict)
     timeout: int = 600
     memory: int = 4096
     cpu: float = 2.0

synth_ai/task/config.py

@@ -257,3 +257,5 @@ class FilterConfig:
         output_path.parent.mkdir(parents=True, exist_ok=True)
         return output_path
 
+
+