synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/sdk/baseline/__init__.py

@@ -0,0 +1,25 @@
+"""Baseline file system for self-contained task evaluation.
+
+This package provides abstractions for defining and executing baseline evaluations
+without requiring deployed task apps. Supports both class-based and function-based
+task runners with first-class train/val/test split support.
+"""
+
+from __future__ import annotations
+
+from synth_ai.sdk.baseline.config import (
+    BaselineConfig,
+    BaselineResults,
+    BaselineTaskRunner,
+    DataSplit,
+    TaskResult,
+)
+
+__all__ = [
+    "BaselineConfig",
+    "BaselineTaskRunner",
+    "DataSplit",
+    "TaskResult",
+    "BaselineResults",
+]
+

synth_ai/sdk/baseline/config.py

@@ -0,0 +1,209 @@
+"""Core dataclasses for baseline configuration and results."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+
+class BaselineTaskRunner:
+    """
+    Base class for task runners.
+    
+    Subclasses should implement `run_task` method for class-based approach,
+    or you can use standalone async functions for function-based approach.
+    """
+    
+    def __init__(
+        self,
+        policy_config: Dict[str, Any],
+        env_config: Dict[str, Any],
+    ):
+        """
+        Initialize task runner with configuration.
+        
+        Args:
+            policy_config: Policy configuration (model, temperature, etc.)
+            env_config: Environment configuration (max_steps, difficulty, etc.)
+        """
+        self.policy_config = policy_config
+        self.env_config = env_config
+    
+    async def run_task(self, seed: int) -> TaskResult:
+        """
+        Execute a single task instance.
+        
+        This method is called for each seed in the selected split.
+        
+        Args:
+            seed: The seed/index for this task instance
+        
+        Returns:
+            TaskResult: Structured result containing success, rewards, metadata, trace
+        """
+        raise NotImplementedError("Subclasses must implement run_task method")
+
+
+@dataclass
+class DataSplit:
+    """Definition of a data split (train/val/test)."""
+    
+    name: str  # "train", "val", "test"
+    seeds: List[int]  # Seed/index values for this split
+    metadata: Dict[str, Any] = field(default_factory=dict)  # Optional metadata
+
+
+@dataclass
+class TaskResult:
+    """Result from a single task execution."""
+    
+    # Required: Seed/index that was evaluated
+    seed: int
+    
+    # Required: Did the task complete successfully?
+    success: bool
+    
+    # Required: Outcome reward for the episode
+    outcome_reward: float
+    
+    # Optional: Event rewards (step-level)
+    event_rewards: List[Dict[str, Any]] = field(default_factory=list)
+    
+    # Optional: Total steps/turns taken
+    total_steps: int = 0
+    
+    # Optional: Metadata (achievements, completion info, etc.)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Error information if success=False
+    error: Optional[str] = None
+    
+    # Optional: v3 trace (SessionTrace dict)
+    trace: Optional[Dict[str, Any]] = None
+
+
+# Type alias for task runner (can be class or function)
+TaskRunnerType = (
+    type[BaselineTaskRunner]
+    | Callable[[int, dict[str, Any], dict[str, Any]], Any]  # Function signature
+)
+
+# Type alias for result aggregator (can be class or function)
+AggregatorType = (
+    type[Any]  # Class with aggregate() method
+    | Callable[[list[TaskResult]], dict[str, Any]]  # Function signature
+)
+
+
+@dataclass
+class BaselineConfig:
+    """Configuration for a baseline file.
+    
+    A baseline file defines how to evaluate a task without requiring
+    a deployed task app. It provides self-contained evaluation logic
+    with first-class support for train/val/test splits.
+    
+    Supports both class-based and function-based task runners:
+    - Class-based: Pass a class that inherits from BaselineTaskRunner
+    - Function-based: Pass an async function with signature:
+      async def task_runner(seed: int, policy_config: Dict[str, Any], 
+                           env_config: Dict[str, Any]) -> TaskResult
+    """
+    
+    # Required: Unique identifier for this baseline config
+    baseline_id: str
+    
+    # Required: Human-readable name
+    name: str
+    
+    # Required: Task runner (class or function)
+    # Class-based: Pass a class inheriting from BaselineTaskRunner
+    #   The class will be instantiated with policy_config and env_config,
+    #   and run_task(seed) will be called for each seed.
+    # Function-based: Pass an async function with signature:
+    #   async def task_runner(seed: int, policy_config: Dict[str, Any], 
+    #                        env_config: Dict[str, Any]) -> TaskResult
+    task_runner: TaskRunnerType
+    
+    # Required: Data splits (train/val/test)
+    splits: Dict[str, DataSplit]
+    
+    # Optional: Description for documentation
+    description: str = ""
+    
+    # Optional: Default policy configuration
+    default_policy_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Default environment configuration
+    default_env_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for filtering/organization
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Tags for filtering and discovery
+    tags: List[str] = field(default_factory=list)
+    
+    # Optional: Custom result aggregator (class or function)
+    # Class-based: Pass a class with aggregate(results: List[TaskResult]) method
+    #   The class will be instantiated and aggregate() called.
+    # Function-based: Pass a function with signature:
+    #   def aggregate_results(results: List[TaskResult]) -> Dict[str, Any]
+    result_aggregator: Optional[AggregatorType] = None
+    
+    # Optional: Path to this baseline file (set by discovery)
+    _source_path: Optional[Path] = None
+    
+    def matches_tag(self, tag: str) -> bool:
+        """Check if baseline matches a tag (case-insensitive)."""
+        return tag.lower() in [t.lower() for t in self.tags]
+    
+    def matches_metadata(self, key: str, value: Any) -> bool:
+        """Check if baseline metadata matches key-value pair."""
+        return self.metadata.get(key) == value
+
+
+@dataclass
+class BaselineResults:
+    """Aggregate results from a baseline evaluation."""
+    
+    # Configuration that was used
+    config: BaselineConfig
+    
+    # Split that was evaluated
+    split_name: str
+    
+    # Per-seed results
+    results: List[TaskResult]
+    
+    # Aggregate metrics
+    aggregate_metrics: Dict[str, Any]
+    
+    # Execution metadata
+    execution_time_seconds: float
+    model_name: str
+    timestamp: str
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary for JSON output."""
+        return {
+            "baseline_id": self.config.baseline_id,
+            "name": self.config.name,
+            "split": self.split_name,
+            "model": self.model_name,
+            "timestamp": self.timestamp,
+            "execution_time_seconds": self.execution_time_seconds,
+            "aggregate_metrics": self.aggregate_metrics,
+            "results": [
+                {
+                    "seed": r.seed,
+                    "success": r.success,
+                    "outcome_reward": r.outcome_reward,
+                    "total_steps": r.total_steps,
+                    "metadata": r.metadata,
+                    "error": r.error,
+                }
+                for r in self.results
+            ],
+        }
+

synth_ai/sdk/baseline/discovery.py

@@ -0,0 +1,216 @@
+"""AST-based discovery mechanism for baseline files."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from synth_ai.sdk.baseline.config import BaselineConfig
+
+# Search patterns for baseline files
+BASELINE_FILE_PATTERNS = [
+    "**/baseline/*.py",
+    "**/baselines/*.py",
+    "**/*_baseline.py",
+]
+
+# Directories to ignore during discovery
+IGNORE_PATTERNS = {
+    "__pycache__",
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "build",
+    "dist",
+    ".mypy_cache",
+    ".pytest_cache",
+}
+
+
+@dataclass
+class BaselineChoice:
+    """Represents a discovered baseline configuration."""
+    
+    baseline_id: str
+    path: Path
+    lineno: int
+    source: str  # "discovered" or "registered"
+    config: Optional[BaselineConfig] = None
+
+
+class BaselineConfigVisitor(ast.NodeVisitor):
+    """AST visitor to find BaselineConfig instances."""
+    
+    def __init__(self):
+        self.matches: List[Tuple[str, int]] = []  # (baseline_id, lineno)
+    
+    def visit_Assign(self, node: ast.Assign) -> None:
+        """Visit assignment statements looking for BaselineConfig."""
+        if not isinstance(node.value, ast.Call):
+            self.generic_visit(node)
+            return
+        
+        # Check if right-hand side is BaselineConfig(...)
+        func = node.value.func
+        if isinstance(func, ast.Name) and func.id == "BaselineConfig":
+            # Extract baseline_id from constructor args
+            baseline_id = self._extract_baseline_id(node.value)
+            if baseline_id:
+                self.matches.append((baseline_id, node.lineno))
+        
+        self.generic_visit(node)
+    
+    def _extract_baseline_id(self, call_node: ast.Call) -> Optional[str]:
+        """Extract baseline_id from BaselineConfig constructor."""
+        for keyword in call_node.keywords:
+            if keyword.arg == "baseline_id" and isinstance(keyword.value, ast.Constant):
+                val = keyword.value.value
+                if isinstance(val, str):
+                    return val
+        return None
+
+
+def should_ignore_path(path: Path) -> bool:
+    """Check if a path should be ignored during discovery."""
+    return any(part in IGNORE_PATTERNS for part in path.parts)
+
+
+def discover_baseline_files(search_roots: List[Path]) -> List[BaselineChoice]:
+    """Discover baseline files via AST scanning.
+    
+    Args:
+        search_roots: List of root directories to search in
+    
+    Returns:
+        List of BaselineChoice objects representing discovered baselines
+    """
+    results: List[BaselineChoice] = []
+    seen = set()
+    
+    for root in search_roots:
+        if not root.exists():
+            continue
+        
+        for pattern in BASELINE_FILE_PATTERNS:
+            for path in root.glob(pattern):
+                if should_ignore_path(path):
+                    continue
+                
+                try:
+                    source = path.read_text(encoding="utf-8")
+                    tree = ast.parse(source, filename=str(path))
+                except (OSError, SyntaxError):
+                    continue
+                
+                visitor = BaselineConfigVisitor()
+                visitor.visit(tree)
+                
+                for baseline_id, lineno in visitor.matches:
+                    key = (baseline_id, path.resolve())
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    
+                    results.append(
+                        BaselineChoice(
+                            baseline_id=baseline_id,
+                            path=path.resolve(),
+                            lineno=lineno,
+                            source="discovered",
+                        )
+                    )
+    
+    return results
+
+
+def load_baseline_config_from_file(
+    baseline_id: str,
+    path: Path,
+) -> BaselineConfig:
+    """Load a BaselineConfig from a Python file.
+    
+    Args:
+        baseline_id: The baseline_id to look for
+        path: Path to the Python file
+    
+    Returns:
+        BaselineConfig instance
+    
+    Raises:
+        ValueError: If baseline_id not found or file cannot be loaded
+    """
+    # Load the module
+    spec = importlib.util.spec_from_file_location("baseline_module", path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Cannot load baseline file: {path}")
+    
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except ModuleNotFoundError as e:
+        missing_module = str(e).split("'")[1] if "'" in str(e) else str(e)
+        raise ImportError(
+            f"❌ Missing dependency for baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Missing module: {missing_module}\n"
+            f"   Fix: pip install {missing_module}  (or 'uv add {missing_module}')"
+        ) from e
+    except SyntaxError as e:
+        raise ValueError(
+            f"❌ Syntax error in baseline file '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error at line {e.lineno}: {e.msg}\n"
+            f"   Text: {e.text.strip() if e.text else 'N/A'}\n"
+            f"   Fix: Check the Python syntax in the baseline file"
+        ) from e
+    except Exception as e:
+        error_type = type(e).__name__
+        raise ValueError(
+            f"❌ Failed to load baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error type: {error_type}\n"
+            f"   Message: {str(e)}\n"
+            f"   This may be due to:\n"
+            f"     - Missing dependencies (check imports)\n"
+            f"     - Configuration errors in the baseline file\n"
+            f"     - Environment variables not set\n"
+            f"   Tip: Run with --verbose for more details"
+        ) from e
+    
+    # Find the BaselineConfig instance
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig) and attr.baseline_id == baseline_id:
+            # Set source path for reference
+            attr._source_path = path
+            return attr
+    
+    # Provide helpful error message
+    found_configs = []
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig):
+            found_configs.append(attr.baseline_id)
+    
+    if found_configs:
+        raise ValueError(
+            f"❌ Baseline '{baseline_id}' not found in {path}\n"
+            f"   Found baselines in this file: {', '.join(found_configs)}\n"
+            f"   Fix: Use one of the above baseline IDs or check the baseline_id parameter"
+        )
+    else:
+        raise ValueError(
+            f"❌ No BaselineConfig instances found in {path}\n"
+            f"   Expected to find a BaselineConfig with baseline_id='{baseline_id}'\n"
+            f"   Fix: Ensure the file defines a BaselineConfig instance with baseline_id='{baseline_id}'"
+        )
+

synth_ai/sdk/baseline/execution.py

@@ -0,0 +1,154 @@
+"""Execution engine for baseline evaluations."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from synth_ai.sdk.baseline.config import (
+    BaselineConfig,
+    BaselineTaskRunner,
+    TaskResult,
+)
+
+
+def default_aggregator(results: List[TaskResult]) -> Dict[str, Any]:
+    """Default result aggregation function.
+    
+    Computes mean, std, min, max, success rate, and other basic metrics.
+    
+    Args:
+        results: List of TaskResult objects from all seeds
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    successful_results = [r for r in results if r.success]
+    outcome_rewards = [r.outcome_reward for r in successful_results]
+    
+    if not outcome_rewards:
+        return {
+            "mean_outcome_reward": 0.0,
+            "std_outcome_reward": 0.0,
+            "min_outcome_reward": 0.0,
+            "max_outcome_reward": 0.0,
+            "success_rate": 0.0,
+            "total_tasks": len(results),
+            "successful_tasks": 0,
+            "failed_tasks": len(results),
+        }
+    
+    mean_reward = sum(outcome_rewards) / len(outcome_rewards)
+    
+    # Calculate standard deviation
+    variance = sum((x - mean_reward) ** 2 for x in outcome_rewards) / len(outcome_rewards)
+    std_reward = variance ** 0.5
+    
+    return {
+        "mean_outcome_reward": mean_reward,
+        "std_outcome_reward": std_reward,
+        "min_outcome_reward": min(outcome_rewards),
+        "max_outcome_reward": max(outcome_rewards),
+        "success_rate": len(successful_results) / len(results),
+        "total_tasks": len(results),
+        "successful_tasks": len(successful_results),
+        "failed_tasks": len(results) - len(successful_results),
+    }
+
+
+def _is_class_based_runner(task_runner: Any) -> bool:
+    """Check if task_runner is a class (not a function)."""
+    return (
+        isinstance(task_runner, type)
+        and issubclass(task_runner, BaselineTaskRunner)
+    )
+
+
+async def run_baseline_evaluation(
+    config: BaselineConfig,
+    seeds: List[int],
+    policy_config: Dict[str, Any],
+    env_config: Dict[str, Any],
+    concurrency: int = 4,
+) -> List[TaskResult]:
+    """Run baseline evaluation for given seeds.
+    
+    Args:
+        config: BaselineConfig instance
+        seeds: List of seeds to evaluate
+        policy_config: Policy configuration (merged from defaults + overrides)
+        env_config: Environment configuration (merged from defaults + overrides)
+        concurrency: Maximum concurrent task executions
+    
+    Returns:
+        List of TaskResult objects, one per seed
+    """
+    # Determine if we're using class-based or function-based runner
+    is_class_based = _is_class_based_runner(config.task_runner)
+    
+    # Instantiate runner if class-based
+    runner_instance: Optional[BaselineTaskRunner] = None
+    if is_class_based:
+        # task_runner is a class - instantiate with policy_config and env_config
+        # as documented in BaselineConfig and BaselineTaskRunner
+        runner_instance = config.task_runner(policy_config, env_config)  # type: ignore[call-arg]
+    
+    # Create semaphore for concurrency control
+    semaphore = asyncio.Semaphore(concurrency)
+    
+    async def run_task(seed: int) -> TaskResult:
+        """Execute a single task with error handling."""
+        async with semaphore:
+            try:
+                if is_class_based and runner_instance:
+                    # Class-based: call run_task method
+                    return await runner_instance.run_task(seed)
+                else:
+                    # Function-based: call function directly
+                    task_runner_fn = config.task_runner
+                    if callable(task_runner_fn):
+                        result = task_runner_fn(seed, policy_config, env_config)  # type: ignore[call-arg]
+                        # Handle both sync and async functions
+                        if hasattr(result, "__await__"):
+                            return await result
+                        return result
+                    raise RuntimeError("task_runner is not callable")
+            except Exception as exc:
+                # Return error result
+                return TaskResult(
+                    seed=seed,
+                    success=False,
+                    outcome_reward=0.0,
+                    error=str(exc),
+                )
+    
+    # Execute all tasks concurrently
+    results = await asyncio.gather(*[run_task(seed) for seed in seeds])
+    return list(results)
+
+
+def aggregate_results(
+    config: BaselineConfig,
+    results: List[TaskResult],
+) -> Dict[str, Any]:
+    """Aggregate results using custom aggregator or default.
+    
+    Args:
+        config: BaselineConfig instance
+        results: List of TaskResult objects
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    if config.result_aggregator is None:
+        return default_aggregator(results)
+    
+    # Check if aggregator is a class or function
+    if isinstance(config.result_aggregator, type):
+        # Class-based: instantiate and call aggregate()
+        aggregator_instance = config.result_aggregator()
+        return aggregator_instance.aggregate(results)
+    else:
+        # Function-based: call directly
+        return config.result_aggregator(results)
+

synth_ai/sdk/graphs/__init__.py

@@ -0,0 +1,15 @@
+from .completions import (
+    GraphCompletionsClient,
+    GraphInfo,
+    GraphTarget,
+    ListGraphsResponse,
+    VerifierClient,
+)
+
+__all__ = [
+    "GraphCompletionsClient",
+    "GraphInfo",
+    "GraphTarget",
+    "ListGraphsResponse",
+    "VerifierClient",
+]