synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/api/tunnel.py

@@ -0,0 +1,49 @@
+"""
+Backend API client for Cloudflare Tunnel provisioning.
+"""
+import os
+from typing import Any, Dict, Optional
+
+import httpx
+
+# Default backend URL - can be overridden via SYNTH_BACKEND_BASE_URL
+BACKEND_BASE_URL = os.getenv("SYNTH_BACKEND_BASE_URL", "https://api.usesynth.ai")
+
+
+async def create_tunnel(
+    synth_api_key: str,
+    port: int,
+    subdomain: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Create a managed Cloudflare tunnel via Synth backend API.
+    
+    Args:
+        synth_api_key: Synth API key for authentication
+        port: Local port the tunnel will forward to
+        subdomain: Optional custom subdomain (e.g., "my-company")
+    
+    Returns:
+        Dict containing:
+        - tunnel_token: Token for cloudflared
+        - hostname: Public hostname (e.g., "cust-abc123.usesynth.ai")
+        - access_client_id: Cloudflare Access client ID (if Access enabled)
+        - access_client_secret: Cloudflare Access client secret (if Access enabled)
+    
+    Raises:
+        httpx.HTTPStatusError: If API request fails
+    """
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        response = await client.post(
+            f"{BACKEND_BASE_URL}/api/v1/tunnels",
+            headers={"Authorization": f"Bearer {synth_api_key}"},
+            json={
+                "port": port,
+                "subdomain": subdomain,
+                "mode": "managed",
+                "enable_access": True,
+            },
+        )
+        response.raise_for_status()
+        return response.json()
+

synth_ai/auth/credentials.py

@@ -0,0 +1,94 @@
+import contextlib
+import time
+import webbrowser
+from dataclasses import dataclass
+
+import requests
+from requests import RequestException
+
+ORIGIN = "https://www.usesynth.ai"
+INIT_URL = ORIGIN + "/api/sdk/handshake/init"
+TOKEN_URL = ORIGIN + "/api/sdk/handshake/token"
+POLL_INTERVAL = 3
+
+
+@dataclass
+class AuthSession:
+    device_code: str
+    verification_uri: str
+    expires_at: float
+
+
+def init_auth_session() -> AuthSession:    
+    try:
+        res = requests.post(INIT_URL, timeout=10)
+    except RequestException as exc:
+        raise RuntimeError(f"Failed to reach handshake init endpoint: {exc}") from exc
+    if res.status_code != 200:
+        body = res.text.strip()
+        raise RuntimeError(f"Handshake init failed ({res.status_code}): {body or 'no response body'}")
+
+    try:
+        data = res.json()
+    except ValueError as err:
+        raise RuntimeError("Handshake init returned malformed JSON.") from err
+
+    device_code = str(data.get("device_code") or "").strip()
+    verification_uri = str(data.get("verification_uri") or "").strip()
+    expires_in = int(data.get("expires_in") or 600)
+    if not device_code or not verification_uri or not expires_in:
+        raise RuntimeError("Handshake init response missing device_code or verification_uri or expires_in.")
+
+    return AuthSession(
+        device_code=device_code,
+        verification_uri=verification_uri,
+        expires_at=time.time() + expires_in
+    )
+
+
+def fetch_data(device_code: str) -> requests.Response | None:
+    try:
+        return requests.post(
+            TOKEN_URL,
+            json={"device_code": device_code},
+            timeout=10,
+        )
+    except RequestException:
+        return None
+
+
+def fetch_credentials_from_web_browser() -> dict:
+    print(f"Fetching your credentials from {ORIGIN}")
+
+    auth_session = init_auth_session()
+
+    with contextlib.suppress(Exception):
+        webbrowser.open(auth_session.verification_uri)
+
+    data = None
+    while time.time() <= auth_session.expires_at:
+        res = fetch_data(auth_session.device_code)
+        if not res:
+            time.sleep(POLL_INTERVAL)
+            continue
+        if res.status_code == 200:
+            try:
+                data = res.json()
+            except ValueError as err:
+                raise RuntimeError("Handshake token returned malformed JSON.") from err
+            break
+        if res.status_code in (404, 410):
+            raise RuntimeError("Handshake failed: device code expired or was revoked.")
+        time.sleep(POLL_INTERVAL)
+    if data is None:
+        raise TimeoutError("Handshake timed out before credentials were returned.")
+
+    print(f"Connected to {ORIGIN}")
+    credentials = data.get("keys")
+    if not isinstance(credentials, dict):
+        credentials = {}
+
+    return {
+        "SYNTH_API_KEY": str(credentials.get("synth") or "").strip(),
+        "ENVIRONMENT_API_KEY": str(credentials.get("rl_env") or "").strip()
+    }

synth_ai/baseline/__init__.py

@@ -0,0 +1,25 @@
+"""Baseline file system for self-contained task evaluation.
+
+This package provides abstractions for defining and executing baseline evaluations
+without requiring deployed task apps. Supports both class-based and function-based
+task runners with first-class train/val/test split support.
+"""
+
+from __future__ import annotations
+
+from synth_ai.baseline.config import (
+    BaselineConfig,
+    BaselineResults,
+    BaselineTaskRunner,
+    DataSplit,
+    TaskResult,
+)
+
+__all__ = [
+    "BaselineConfig",
+    "BaselineTaskRunner",
+    "DataSplit",
+    "TaskResult",
+    "BaselineResults",
+]
+

synth_ai/baseline/config.py

@@ -0,0 +1,209 @@
+"""Core dataclasses for baseline configuration and results."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+
+class BaselineTaskRunner:
+    """
+    Base class for task runners.
+    
+    Subclasses should implement `run_task` method for class-based approach,
+    or you can use standalone async functions for function-based approach.
+    """
+    
+    def __init__(
+        self,
+        policy_config: Dict[str, Any],
+        env_config: Dict[str, Any],
+    ):
+        """
+        Initialize task runner with configuration.
+        
+        Args:
+            policy_config: Policy configuration (model, temperature, etc.)
+            env_config: Environment configuration (max_steps, difficulty, etc.)
+        """
+        self.policy_config = policy_config
+        self.env_config = env_config
+    
+    async def run_task(self, seed: int) -> TaskResult:
+        """
+        Execute a single task instance.
+        
+        This method is called for each seed in the selected split.
+        
+        Args:
+            seed: The seed/index for this task instance
+        
+        Returns:
+            TaskResult: Structured result containing success, rewards, metadata, trace
+        """
+        raise NotImplementedError("Subclasses must implement run_task method")
+
+
+@dataclass
+class DataSplit:
+    """Definition of a data split (train/val/test)."""
+    
+    name: str  # "train", "val", "test"
+    seeds: List[int]  # Seed/index values for this split
+    metadata: Dict[str, Any] = field(default_factory=dict)  # Optional metadata
+
+
+@dataclass
+class TaskResult:
+    """Result from a single task execution."""
+    
+    # Required: Seed/index that was evaluated
+    seed: int
+    
+    # Required: Did the task complete successfully?
+    success: bool
+    
+    # Required: Outcome reward for the episode
+    outcome_reward: float
+    
+    # Optional: Event rewards (step-level)
+    event_rewards: List[Dict[str, Any]] = field(default_factory=list)
+    
+    # Optional: Total steps/turns taken
+    total_steps: int = 0
+    
+    # Optional: Metadata (achievements, completion info, etc.)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Error information if success=False
+    error: Optional[str] = None
+    
+    # Optional: v3 trace (SessionTrace dict)
+    trace: Optional[Dict[str, Any]] = None
+
+
+# Type alias for task runner (can be class or function)
+TaskRunnerType = (
+    type[BaselineTaskRunner]
+    | Callable[[int, dict[str, Any], dict[str, Any]], Any]  # Function signature
+)
+
+# Type alias for result aggregator (can be class or function)
+AggregatorType = (
+    type[Any]  # Class with aggregate() method
+    | Callable[[list[TaskResult]], dict[str, Any]]  # Function signature
+)
+
+
+@dataclass
+class BaselineConfig:
+    """Configuration for a baseline file.
+    
+    A baseline file defines how to evaluate a task without requiring
+    a deployed task app. It provides self-contained evaluation logic
+    with first-class support for train/val/test splits.
+    
+    Supports both class-based and function-based task runners:
+    - Class-based: Pass a class that inherits from BaselineTaskRunner
+    - Function-based: Pass an async function with signature:
+      async def task_runner(seed: int, policy_config: Dict[str, Any], 
+                           env_config: Dict[str, Any]) -> TaskResult
+    """
+    
+    # Required: Unique identifier for this baseline config
+    baseline_id: str
+    
+    # Required: Human-readable name
+    name: str
+    
+    # Required: Task runner (class or function)
+    # Class-based: Pass a class inheriting from BaselineTaskRunner
+    #   The class will be instantiated with policy_config and env_config,
+    #   and run_task(seed) will be called for each seed.
+    # Function-based: Pass an async function with signature:
+    #   async def task_runner(seed: int, policy_config: Dict[str, Any], 
+    #                        env_config: Dict[str, Any]) -> TaskResult
+    task_runner: TaskRunnerType
+    
+    # Required: Data splits (train/val/test)
+    splits: Dict[str, DataSplit]
+    
+    # Optional: Description for documentation
+    description: str = ""
+    
+    # Optional: Default policy configuration
+    default_policy_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Default environment configuration
+    default_env_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for filtering/organization
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Tags for filtering and discovery
+    tags: List[str] = field(default_factory=list)
+    
+    # Optional: Custom result aggregator (class or function)
+    # Class-based: Pass a class with aggregate(results: List[TaskResult]) method
+    #   The class will be instantiated and aggregate() called.
+    # Function-based: Pass a function with signature:
+    #   def aggregate_results(results: List[TaskResult]) -> Dict[str, Any]
+    result_aggregator: Optional[AggregatorType] = None
+    
+    # Optional: Path to this baseline file (set by discovery)
+    _source_path: Optional[Path] = None
+    
+    def matches_tag(self, tag: str) -> bool:
+        """Check if baseline matches a tag (case-insensitive)."""
+        return tag.lower() in [t.lower() for t in self.tags]
+    
+    def matches_metadata(self, key: str, value: Any) -> bool:
+        """Check if baseline metadata matches key-value pair."""
+        return self.metadata.get(key) == value
+
+
+@dataclass
+class BaselineResults:
+    """Aggregate results from a baseline evaluation."""
+    
+    # Configuration that was used
+    config: BaselineConfig
+    
+    # Split that was evaluated
+    split_name: str
+    
+    # Per-seed results
+    results: List[TaskResult]
+    
+    # Aggregate metrics
+    aggregate_metrics: Dict[str, Any]
+    
+    # Execution metadata
+    execution_time_seconds: float
+    model_name: str
+    timestamp: str
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary for JSON output."""
+        return {
+            "baseline_id": self.config.baseline_id,
+            "name": self.config.name,
+            "split": self.split_name,
+            "model": self.model_name,
+            "timestamp": self.timestamp,
+            "execution_time_seconds": self.execution_time_seconds,
+            "aggregate_metrics": self.aggregate_metrics,
+            "results": [
+                {
+                    "seed": r.seed,
+                    "success": r.success,
+                    "outcome_reward": r.outcome_reward,
+                    "total_steps": r.total_steps,
+                    "metadata": r.metadata,
+                    "error": r.error,
+                }
+                for r in self.results
+            ],
+        }
+

synth_ai/baseline/discovery.py

@@ -0,0 +1,214 @@
+"""AST-based discovery mechanism for baseline files."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from synth_ai.baseline.config import BaselineConfig
+
+# Search patterns for baseline files
+BASELINE_FILE_PATTERNS = [
+    "**/baseline/*.py",
+    "**/baselines/*.py",
+    "**/*_baseline.py",
+]
+
+# Directories to ignore during discovery
+IGNORE_PATTERNS = {
+    "__pycache__",
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "build",
+    "dist",
+    ".mypy_cache",
+    ".pytest_cache",
+}
+
+
+@dataclass
+class BaselineChoice:
+    """Represents a discovered baseline configuration."""
+    
+    baseline_id: str
+    path: Path
+    lineno: int
+    source: str  # "discovered" or "registered"
+    config: Optional[BaselineConfig] = None
+
+
+class BaselineConfigVisitor(ast.NodeVisitor):
+    """AST visitor to find BaselineConfig instances."""
+    
+    def __init__(self):
+        self.matches: List[Tuple[str, int]] = []  # (baseline_id, lineno)
+    
+    def visit_Assign(self, node: ast.Assign) -> None:
+        """Visit assignment statements looking for BaselineConfig."""
+        if not isinstance(node.value, ast.Call):
+            self.generic_visit(node)
+            return
+        
+        # Check if right-hand side is BaselineConfig(...)
+        func = node.value.func
+        if isinstance(func, ast.Name) and func.id == "BaselineConfig":
+            # Extract baseline_id from constructor args
+            baseline_id = self._extract_baseline_id(node.value)
+            if baseline_id:
+                self.matches.append((baseline_id, node.lineno))
+        
+        self.generic_visit(node)
+    
+    def _extract_baseline_id(self, call_node: ast.Call) -> Optional[str]:
+        """Extract baseline_id from BaselineConfig constructor."""
+        for keyword in call_node.keywords:
+            if keyword.arg == "baseline_id" and isinstance(keyword.value, ast.Constant):
+                return keyword.value.value
+        return None
+
+
+def should_ignore_path(path: Path) -> bool:
+    """Check if a path should be ignored during discovery."""
+    return any(part in IGNORE_PATTERNS for part in path.parts)
+
+
+def discover_baseline_files(search_roots: List[Path]) -> List[BaselineChoice]:
+    """Discover baseline files via AST scanning.
+    
+    Args:
+        search_roots: List of root directories to search in
+    
+    Returns:
+        List of BaselineChoice objects representing discovered baselines
+    """
+    results: List[BaselineChoice] = []
+    seen = set()
+    
+    for root in search_roots:
+        if not root.exists():
+            continue
+        
+        for pattern in BASELINE_FILE_PATTERNS:
+            for path in root.glob(pattern):
+                if should_ignore_path(path):
+                    continue
+                
+                try:
+                    source = path.read_text(encoding="utf-8")
+                    tree = ast.parse(source, filename=str(path))
+                except (OSError, SyntaxError):
+                    continue
+                
+                visitor = BaselineConfigVisitor()
+                visitor.visit(tree)
+                
+                for baseline_id, lineno in visitor.matches:
+                    key = (baseline_id, path.resolve())
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    
+                    results.append(
+                        BaselineChoice(
+                            baseline_id=baseline_id,
+                            path=path.resolve(),
+                            lineno=lineno,
+                            source="discovered",
+                        )
+                    )
+    
+    return results
+
+
+def load_baseline_config_from_file(
+    baseline_id: str,
+    path: Path,
+) -> BaselineConfig:
+    """Load a BaselineConfig from a Python file.
+    
+    Args:
+        baseline_id: The baseline_id to look for
+        path: Path to the Python file
+    
+    Returns:
+        BaselineConfig instance
+    
+    Raises:
+        ValueError: If baseline_id not found or file cannot be loaded
+    """
+    # Load the module
+    spec = importlib.util.spec_from_file_location("baseline_module", path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Cannot load baseline file: {path}")
+    
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except ModuleNotFoundError as e:
+        missing_module = str(e).split("'")[1] if "'" in str(e) else str(e)
+        raise ImportError(
+            f"❌ Missing dependency for baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Missing module: {missing_module}\n"
+            f"   Fix: pip install {missing_module}  (or 'uv add {missing_module}')"
+        ) from e
+    except SyntaxError as e:
+        raise ValueError(
+            f"❌ Syntax error in baseline file '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error at line {e.lineno}: {e.msg}\n"
+            f"   Text: {e.text.strip() if e.text else 'N/A'}\n"
+            f"   Fix: Check the Python syntax in the baseline file"
+        ) from e
+    except Exception as e:
+        error_type = type(e).__name__
+        raise ValueError(
+            f"❌ Failed to load baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error type: {error_type}\n"
+            f"   Message: {str(e)}\n"
+            f"   This may be due to:\n"
+            f"     - Missing dependencies (check imports)\n"
+            f"     - Configuration errors in the baseline file\n"
+            f"     - Environment variables not set\n"
+            f"   Tip: Run with --verbose for more details"
+        ) from e
+    
+    # Find the BaselineConfig instance
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig) and attr.baseline_id == baseline_id:
+            # Set source path for reference
+            attr._source_path = path
+            return attr
+    
+    # Provide helpful error message
+    found_configs = []
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig):
+            found_configs.append(attr.baseline_id)
+    
+    if found_configs:
+        raise ValueError(
+            f"❌ Baseline '{baseline_id}' not found in {path}\n"
+            f"   Found baselines in this file: {', '.join(found_configs)}\n"
+            f"   Fix: Use one of the above baseline IDs or check the baseline_id parameter"
+        )
+    else:
+        raise ValueError(
+            f"❌ No BaselineConfig instances found in {path}\n"
+            f"   Expected to find a BaselineConfig with baseline_id='{baseline_id}'\n"
+            f"   Fix: Ensure the file defines a BaselineConfig instance with baseline_id='{baseline_id}'"
+        )
+