synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/task/__init__.py

@@ -1,121 +0,0 @@
-from .auth import (
-    is_api_key_header_authorized,
-    normalize_environment_api_key,
-    require_api_key_dependency,
-)
-from .client import TaskAppClient
-from .config import EvalConfig, FilterConfig
-from .contracts import (
-    DatasetInfo,
-    InferenceInfo,
-    LimitsInfo,
-    RolloutEnvSpec,
-    RolloutMetrics,
-    RolloutPolicySpec,
-    RolloutRecordConfig,
-    RolloutRequest,
-    RolloutResponse,
-    RolloutSafetyConfig,
-    RolloutStep,
-    RolloutTrajectory,
-    RubricInfo,
-    RubricSection,
-    TaskAppEndpoints,
-    TaskDescriptor,
-    TaskInfo,
-)
-from .datasets import TaskDatasetRegistry, TaskDatasetSpec
-from .errors import error_payload, http_exception, json_error_response
-from .health import task_app_health
-from .json import to_jsonable
-from .proxy import (
-    extract_message_text,
-    inject_system_hint,
-    parse_tool_call_from_text,
-    prepare_for_groq,
-    prepare_for_openai,
-    synthesize_tool_call_if_missing,
-)
-from .rubrics import (
-    Criterion,
-    Rubric,
-    blend_rubrics,
-    load_rubric,
-    score_events_against_rubric,
-    score_outcome_against_rubric,
-)
-from .server import (
-    ProxyConfig,
-    RubricBundle,
-    TaskAppConfig,
-    create_task_app,
-    run_task_app,
-)
-from .validators import (
-    normalize_inference_url,
-    validate_rollout_response_for_rl,
-    validate_task_app_endpoint,
-    validate_task_app_url,
-)
-from .vendors import (
-    get_groq_key_or_503,
-    get_openai_key_or_503,
-    normalize_vendor_keys,
-)
-
-__all__ = [
-    "normalize_inference_url",
-    "validate_rollout_response_for_rl",
-    "validate_task_app_url",
-    "validate_task_app_endpoint",
-    "task_app_health",
-    "EvalConfig",
-    "FilterConfig",
-    "TaskAppEndpoints",
-    "RolloutEnvSpec",
-    "RolloutPolicySpec",
-    "RolloutRecordConfig",
-    "RolloutSafetyConfig",
-    "RolloutRequest",
-    "RolloutResponse",
-    "RolloutTrajectory",
-    "RolloutStep",
-    "RolloutMetrics",
-    "TaskDescriptor",
-    "DatasetInfo",
-    "RubricInfo",
-    "RubricSection",
-    "InferenceInfo",
-    "LimitsInfo",
-    "TaskInfo",
-    "to_jsonable",
-    "normalize_environment_api_key",
-    "is_api_key_header_authorized",
-    "require_api_key_dependency",
-    "normalize_vendor_keys",
-    "get_openai_key_or_503",
-    "get_groq_key_or_503",
-    "prepare_for_openai",
-    "prepare_for_groq",
-    "inject_system_hint",
-    "extract_message_text",
-    "parse_tool_call_from_text",
-    "synthesize_tool_call_if_missing",
-    "TaskDatasetSpec",
-    "TaskDatasetRegistry",
-    "Criterion",
-    "Rubric",
-    "load_rubric",
-    "blend_rubrics",
-    "score_events_against_rubric",
-    "score_outcome_against_rubric",
-    "TaskAppClient",
-    "error_payload",
-    "http_exception",
-    "json_error_response",
-    "run_task_app",
-    "create_task_app",
-    "RubricBundle",
-    "ProxyConfig",
-    "TaskAppConfig",
-]

synth_ai/task/apps/__init__.py

@@ -1,129 +0,0 @@
-"""Registry for Task Apps exposed via the shared FastAPI harness."""
-
-from __future__ import annotations
-
-import importlib
-import sys
-from collections.abc import Callable, Iterable, Sequence
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from ..server import TaskAppConfig
-
-
-@dataclass(slots=True)
-class ModalDeploymentConfig:
-    """Modal deployment defaults for a task app."""
-
-    app_name: str
-    python_version: str = "3.11"
-    pip_packages: Sequence[str] = field(default_factory=tuple)
-    apt_packages: Sequence[str] = field(default_factory=tuple)
-    extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
-    secret_names: Sequence[str] = field(default_factory=tuple)
-    volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)
-    timeout: int = 600
-    memory: int = 4096
-    cpu: float = 2.0
-    min_containers: int = 1
-    max_containers: int = 4
-
-
-@dataclass(slots=True)
-class TaskAppEntry:
-    """Metadata describing a registered task app."""
-
-    app_id: str
-    description: str
-    config_factory: Callable[[], TaskAppConfig]
-    aliases: Sequence[str] = field(default_factory=tuple)
-    env_files: Sequence[str] = field(default_factory=tuple)
-    modal: ModalDeploymentConfig | None = None
-
-
-class TaskAppRegistry:
-    """In-memory registry of known task apps."""
-
-    def __init__(self) -> None:
-        self._entries: dict[str, TaskAppEntry] = {}
-        self._alias_to_id: dict[str, str] = {}
-
-    def register(self, entry: TaskAppEntry) -> None:
-        if entry.app_id in self._entries:
-            raise ValueError(f"Task app already registered: {entry.app_id}")
-        self._entries[entry.app_id] = entry
-        for alias in entry.aliases:
-            if alias in self._alias_to_id:
-                raise ValueError(f"Alias already registered: {alias}")
-            self._alias_to_id[alias] = entry.app_id
-
-    def get(self, app_id: str) -> TaskAppEntry:
-        resolved = self._alias_to_id.get(app_id, app_id)
-        if resolved not in self._entries:
-            raise KeyError(f"Unknown task app id: {app_id}")
-        return self._entries[resolved]
-
-    def list(self) -> list[TaskAppEntry]:
-        return sorted(self._entries.values(), key=lambda entry: entry.app_id)
-
-    def __iter__(self) -> Iterable[TaskAppEntry]:
-        return iter(self.list())
-
-    def clear(self) -> None:
-        """Clear all registered task apps."""
-        self._entries.clear()
-        self._alias_to_id.clear()
-
-
-registry = TaskAppRegistry()
-
-
-def register_task_app(*, entry: TaskAppEntry) -> None:
-    registry.register(entry)
-
-
-def discover_task_apps_from_cwd() -> None:
-    """Discover and register task apps from the current working directory and subdirectories."""
-    cwd = Path.cwd()
-
-    # Look for task app files in common patterns
-    patterns = [
-        "**/task_app/*.py",
-        "**/task_apps/*.py",
-        "**/*_task_app.py",
-        "**/grpo_crafter.py",
-        "**/math_single_step.py",
-    ]
-
-    discovered_files = []
-    for pattern in patterns:
-        discovered_files.extend(cwd.glob(pattern))
-
-    # Add current directory to Python path temporarily
-    original_path = sys.path.copy()
-    try:
-        sys.path.insert(0, str(cwd))
-
-        for file_path in discovered_files:
-            if file_path.name.startswith("__"):
-                continue
-
-            # Convert file path to module name
-            relative_path = file_path.relative_to(cwd)
-            module_parts = list(relative_path.parts[:-1]) + [relative_path.stem]
-            module_name = ".".join(module_parts)
-
-            try:
-                # Import the module to trigger registration
-                importlib.import_module(module_name)
-            except Exception:
-                # Silently skip modules that can't be imported
-                # This allows for graceful handling of missing dependencies
-                continue
-
-    finally:
-        sys.path[:] = original_path
-
-
-# Note: Task apps are now discovered dynamically by the CLI, not auto-registered
-# This allows for better separation between SDK and example-specific implementations

synth_ai/task/client.py

@@ -1,167 +0,0 @@
-"""Async HTTP client for interacting with Task Apps."""
-
-from __future__ import annotations
-
-import asyncio
-import os
-from typing import Any
-
-import httpx
-from pydantic import BaseModel
-
-from .contracts import RolloutRequest, RolloutResponse, TaskInfo
-from .json import to_jsonable
-
-
-def _prepare_payload(payload: Any) -> Any:
-    if payload is None:
-        return None
-    if isinstance(payload, BaseModel):
-        return payload.model_dump(mode="json", by_alias=True)
-    return to_jsonable(payload)
-
-
-class TaskAppClient:
-    def __init__(
-        self,
-        base_url: str,
-        api_key: str | None = None,
-        *,
-        timeout: float = 600.0,
-        retries: int = 3,
-    ) -> None:
-        self.base_url = base_url.rstrip("/")
-        self.api_key = api_key
-        self.timeout = timeout
-        self.retries = max(1, retries)
-        self._client: httpx.AsyncClient | None = None
-        self.env = _TaskAppEnvironmentClient(self)
-
-    async def __aenter__(self) -> TaskAppClient:
-        await self._ensure_client()
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb) -> None:
-        await self.aclose()
-
-    async def _ensure_client(self) -> httpx.AsyncClient:
-        if self._client is None:
-            self._client = httpx.AsyncClient(
-                base_url=self.base_url,
-                timeout=httpx.Timeout(self.timeout),
-                follow_redirects=True,
-            )
-        return self._client
-
-    def _headers(self) -> dict[str, str]:
-        headers: dict[str, str] = {}
-        # Primary key
-        primary = (self.api_key or "").strip()
-        if primary:
-            headers["X-API-Key"] = primary
-            # Also set Authorization for clients that read bearer tokens
-            headers.setdefault("Authorization", f"Bearer {primary}")
-        # Include ALL available environment keys via CSV in X-API-Keys
-        keys: list[str] = []
-        if primary:
-            keys.append(primary)
-        aliases = (os.getenv("ENVIRONMENT_API_KEY_ALIASES") or "").strip()
-        if aliases:
-            for part in aliases.split(","):
-                trimmed = part.strip()
-                if trimmed and trimmed not in keys:
-                    keys.append(trimmed)
-        if keys:
-            headers["X-API-Keys"] = ",".join(keys)
-        return headers
-
-    async def aclose(self) -> None:
-        if self._client is not None:
-            await self._client.aclose()
-            self._client = None
-
-    async def _request(
-        self,
-        method: str,
-        path: str,
-        *,
-        params: dict[str, Any] | list[tuple[str, Any]] | None = None,
-        json_payload: Any = None,
-    ) -> httpx.Response:
-        client = await self._ensure_client()
-        payload = _prepare_payload(json_payload)
-        headers = self._headers()
-        last_exc: Exception | None = None
-        for attempt in range(self.retries):
-            try:
-                response = await client.request(
-                    method,
-                    path,
-                    headers=headers,
-                    params=params,
-                    json=payload,
-                )
-                response.raise_for_status()
-                return response
-            except httpx.HTTPStatusError as exc:
-                if 500 <= exc.response.status_code < 600 and attempt + 1 < self.retries:
-                    await asyncio.sleep(0.1 * (attempt + 1))
-                    last_exc = exc
-                    continue
-                raise
-            except httpx.HTTPError as exc:
-                last_exc = exc
-                if attempt + 1 >= self.retries:
-                    raise
-                await asyncio.sleep(0.1 * (attempt + 1))
-        if last_exc:  # pragma: no cover - defensive
-            raise last_exc
-        raise RuntimeError("Unreachable code in TaskAppClient._request")
-
-    async def health(self) -> dict[str, Any]:
-        response = await self._request("GET", "/health")
-        return response.json()
-
-    async def info(self) -> dict[str, Any]:
-        response = await self._request("GET", "/info")
-        return response.json()
-
-    async def task_info(self, seeds: list[int] | None = None) -> TaskInfo | list[TaskInfo]:
-        params: list[tuple[str, Any]] | None = None
-        if seeds:
-            params = [("seed", seed) for seed in seeds]
-        response = await self._request("GET", "/task_info", params=params)
-        data = response.json()
-        if isinstance(data, list):
-            return [TaskInfo.model_validate(item) for item in data]
-        return TaskInfo.model_validate(data)
-
-    async def rollout(self, request: RolloutRequest) -> RolloutResponse:
-        response = await self._request("POST", "/rollout", json_payload=request)
-        data = response.json()
-        return RolloutResponse.model_validate(data)
-
-
-class _TaskAppEnvironmentClient:
-    def __init__(self, client: TaskAppClient) -> None:
-        self._client = client
-
-    async def initialize(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
-        response = await self._client._request(
-            "POST", f"/env/{env_name}/initialize", json_payload=payload
-        )
-        return response.json()
-
-    async def step(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
-        response = await self._client._request(
-            "POST", f"/env/{env_name}/step", json_payload=payload
-        )
-        return response.json()
-
-    async def terminate(
-        self, env_name: str, payload: dict[str, Any] | None = None
-    ) -> dict[str, Any]:
-        response = await self._client._request(
-            "POST", f"/env/{env_name}/terminate", json_payload=payload or {}
-        )
-        return response.json()

synth_ai/task/config.py

@@ -1,257 +0,0 @@
-"""Configuration dataclasses for task app CLI commands (eval, filter)."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Literal
-
-
-@dataclass(slots=True)
-class EvalConfig:
-    """Configuration for 'synth-ai eval' command.
-    
-    Validates and provides defaults for evaluation runs against task apps.
-    """
-    
-    # Required: Task app identifier
-    app_id: str
-    
-    # Required: Model to evaluate
-    model: str
-    
-    # Required: Seeds to run
-    seeds: list[int]
-    
-    # Optional: Task app URL (None = spawn in-process)
-    task_app_url: str | None = None
-    
-    # Optional: Data split to use
-    split: str = "train"
-    
-    # Optional: Maximum turns/steps per episode
-    max_turns: int | None = None
-    
-    # Optional: Maximum LLM calls per episode
-    max_llm_calls: int = 10
-    
-    # Optional: Concurrency for parallel rollouts
-    concurrency: int = 1
-    
-    # Optional: Environment name
-    env_name: str | None = None
-    
-    # Optional: Policy name
-    policy_name: str | None = None
-    
-    # Optional: Trace format ("compact", "full", "structured")
-    trace_format: Literal["compact", "full", "structured"] = "compact"
-    
-    # Optional: Whether to return traces in response
-    return_trace: bool = False
-    
-    # Optional: Operations sequence (if not provided, generates default)
-    ops: list[str] | None = None
-    
-    # Optional: Environment config overrides
-    env_config: dict[str, Any] = field(default_factory=dict)
-    
-    # Optional: Policy config overrides
-    policy_config: dict[str, Any] = field(default_factory=dict)
-    
-    # Optional: Metadata for traces
-    metadata: dict[str, str] = field(default_factory=dict)
-    
-    # Optional: SQL query for metadata filtering
-    metadata_sql: str | None = None
-    
-    def __post_init__(self):
-        """Validate configuration after initialization."""
-        if not self.app_id:
-            raise ValueError("app_id is required")
-        
-        if not self.model:
-            raise ValueError("model is required")
-        
-        if not self.seeds:
-            raise ValueError("seeds list cannot be empty")
-        
-        if not isinstance(self.seeds, list):
-            raise ValueError("seeds must be a list of integers")
-        
-        if self.concurrency < 1:
-            raise ValueError("concurrency must be >= 1")
-        
-        if self.max_llm_calls < 1:
-            raise ValueError("max_llm_calls must be >= 1")
-        
-        if self.max_turns is not None and self.max_turns < 1:
-            raise ValueError("max_turns must be >= 1")
-        
-        if self.trace_format not in ("compact", "full", "structured"):
-            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
-    
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
-        """Create EvalConfig from a dictionary (e.g. from TOML).
-        
-        Args:
-            data: Dictionary with eval configuration
-            
-        Returns:
-            Validated EvalConfig instance
-        """
-        # Extract known fields
-        config_dict = {
-            "app_id": data.get("app_id"),
-            "model": data.get("model"),
-            "seeds": data.get("seeds", []),
-            "task_app_url": data.get("task_app_url"),
-            "split": data.get("split", "train"),
-            "max_turns": data.get("max_turns"),
-            "max_llm_calls": data.get("max_llm_calls", 10),
-            "concurrency": data.get("concurrency", 1),
-            "env_name": data.get("env_name"),
-            "policy_name": data.get("policy_name"),
-            "trace_format": data.get("trace_format", "compact"),
-            "return_trace": data.get("return_trace", False),
-            "ops": data.get("ops"),
-            "env_config": data.get("env_config", {}),
-            "policy_config": data.get("policy_config", {}),
-            "metadata": data.get("metadata", {}),
-            "metadata_sql": data.get("metadata_sql"),
-        }
-        
-        return cls(**config_dict)
-
-
-@dataclass(slots=True)
-class FilterConfig:
-    """Configuration for 'synth-ai filter' command.
-    
-    Validates and provides defaults for filtering traces into SFT datasets.
-    """
-    
-    # Required: Database path or URL
-    db: str
-    
-    # Required: Output JSONL path
-    output: str
-    
-    # Optional: Filter by data splits
-    splits: list[str] = field(default_factory=list)
-    
-    # Optional: Filter by task IDs
-    task_ids: list[str] = field(default_factory=list)
-    
-    # Optional: Filter by models
-    models: list[str] = field(default_factory=list)
-    
-    # Optional: Minimum official score threshold
-    min_official_score: float | None = None
-    
-    # Optional: Maximum official score threshold
-    max_official_score: float | None = None
-    
-    # Optional: Minimum judge scores (judge_name -> min_score)
-    min_judge_scores: dict[str, float] = field(default_factory=dict)
-    
-    # Optional: Maximum judge scores (judge_name -> max_score)
-    max_judge_scores: dict[str, float] = field(default_factory=dict)
-    
-    # Optional: Limit number of examples
-    limit: int | None = None
-    
-    # Optional: Offset for pagination
-    offset: int | None = None
-    
-    # Optional: Whether to shuffle results
-    shuffle: bool = False
-    
-    # Optional: Random seed for shuffling
-    shuffle_seed: int | None = None
-    
-    def __post_init__(self):
-        """Validate configuration after initialization."""
-        if not self.db:
-            raise ValueError("db (database path or URL) is required")
-        
-        if not self.output:
-            raise ValueError("output (JSONL file path) is required")
-        
-        # Validate output has .jsonl extension
-        output_path = Path(self.output)
-        if output_path.suffix.lower() not in (".jsonl", ".json"):
-            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
-        
-        # Validate score thresholds
-        if self.min_official_score is not None and self.max_official_score is not None:
-            if self.min_official_score > self.max_official_score:
-                raise ValueError("min_official_score cannot be greater than max_official_score")
-        
-        # Validate limit/offset
-        if self.limit is not None and self.limit < 1:
-            raise ValueError("limit must be >= 1")
-        
-        if self.offset is not None and self.offset < 0:
-            raise ValueError("offset must be >= 0")
-        
-        # Validate shuffle seed requires shuffle
-        if self.shuffle_seed is not None and not self.shuffle:
-            raise ValueError("shuffle_seed requires shuffle=true")
-    
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
-        """Create FilterConfig from a dictionary (e.g. from TOML).
-        
-        Args:
-            data: Dictionary with filter configuration
-            
-        Returns:
-            Validated FilterConfig instance
-        """
-        # Extract known fields
-        config_dict = {
-            "db": data.get("db"),
-            "output": data.get("output"),
-            "splits": data.get("splits", []),
-            "task_ids": data.get("task_ids", []),
-            "models": data.get("models", []),
-            "min_official_score": data.get("min_official_score"),
-            "max_official_score": data.get("max_official_score"),
-            "min_judge_scores": data.get("min_judge_scores", {}),
-            "max_judge_scores": data.get("max_judge_scores", {}),
-            "limit": data.get("limit"),
-            "offset": data.get("offset"),
-            "shuffle": data.get("shuffle", False),
-            "shuffle_seed": data.get("shuffle_seed"),
-        }
-        
-        return cls(**config_dict)
-    
-    def get_db_url(self) -> str:
-        """Convert db path to proper SQLite URL if needed.
-        
-        Returns:
-            Database URL suitable for SQLAlchemy/aiosqlite
-        """
-        db_value = self.db.strip()
-        if "://" in db_value:
-            return db_value
-        else:
-            db_path = Path(db_value).expanduser().resolve()
-            # Ensure parent directory exists
-            db_path.parent.mkdir(parents=True, exist_ok=True)
-            return f"sqlite+aiosqlite:///{db_path}"
-    
-    def get_output_path(self) -> Path:
-        """Get resolved output path with parent directory created.
-        
-        Returns:
-            Resolved Path object with parent directory created
-        """
-        output_path = Path(self.output).expanduser().resolve()
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        return output_path
-
-