synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/task/auth.py

@@ -0,0 +1,165 @@
+"""Authentication helpers shared by Task Apps."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable
+from contextlib import suppress
+from typing import Any
+
+from .errors import http_exception
+
+_API_KEY_ENV = "ENVIRONMENT_API_KEY"
+_DEV_API_KEY_ENVS = ("dev_environment_api_key", "DEV_ENVIRONMENT_API_KEY")
+_API_KEY_HEADER = "x-api-key"
+_API_KEYS_HEADER = "x-api-keys"
+_AUTH_HEADER = "authorization"
+_API_KEY_ALIASES_ENV = (
+    "ENVIRONMENT_API_KEY_ALIASES"  # comma-separated list of additional valid keys
+)
+
+
+def _mask(value: str, *, prefix: int = 4) -> str:
+    if not value:
+        return "<empty>"
+    visible = value[:prefix]
+    return f"{visible}{'…' if len(value) > prefix else ''}"
+
+
+def normalize_environment_api_key() -> str | None:
+    """Ensure `ENVIRONMENT_API_KEY` is populated from dev fallbacks.
+
+    Returns the resolved key (if any) so callers can branch on configuration.
+    """
+
+    key = os.getenv(_API_KEY_ENV)
+    if key:
+        return key
+    for env in _DEV_API_KEY_ENVS:
+        candidate = os.getenv(env)
+        if candidate:
+            os.environ[_API_KEY_ENV] = candidate
+            print(
+                f"[task:auth] {_API_KEY_ENV} set from {env} (prefix={_mask(candidate)})",
+                flush=True,
+            )
+            return candidate
+    return None
+
+
+def allowed_environment_api_keys() -> set[str]:
+    """Return the set of valid environment API keys for this Task App.
+
+    Includes:
+    - The primary ENVIRONMENT_API_KEY (normalized from dev fallbacks if needed)
+    - Any comma-separated aliases from ENVIRONMENT_API_KEY_ALIASES
+    """
+    keys: set[str] = set()
+    primary = normalize_environment_api_key()
+    if primary:
+        keys.add(primary)
+    aliases = (os.getenv(_API_KEY_ALIASES_ENV) or "").strip()
+    if aliases:
+        for part in aliases.split(","):
+            trimmed = part.strip()
+            if trimmed:
+                keys.add(trimmed)
+    return keys
+
+
+def _header_values(request: Any, header: str) -> Iterable[str]:
+    header_lower = header.lower()
+    if request is None:
+        return []
+    headers = getattr(request, "headers", None)
+    if headers:
+        raw = headers.get(header) or headers.get(header_lower)
+        if raw is not None:
+            return [raw]
+    if isinstance(request, dict):
+        raw = request.get(header) or request.get(header_lower)
+        if raw is not None:
+            return [raw]
+    # Support passing explicit header dict via keyword arg on FastAPI route handlers
+    for attr in ("headers", "state"):
+        maybe = getattr(request, attr, None)
+        if isinstance(maybe, dict):
+            raw = maybe.get(header) or maybe.get(header_lower)
+            if raw is not None:
+                return [raw]
+    return []
+
+
+def _split_csv(values: Iterable[str]) -> list[str]:
+    seen: list[str] = []
+    for v in values:
+        if not isinstance(v, str):
+            continue
+        for part in v.split(","):
+            trimmed = part.strip()
+            if trimmed:
+                seen.append(trimmed)
+    return seen
+
+
+def is_api_key_header_authorized(request: Any) -> bool:
+    """Return True if any header-provided key matches any allowed environment key."""
+
+    allowed = allowed_environment_api_keys()
+    if not allowed:
+        return False
+    single = list(_header_values(request, _API_KEY_HEADER))
+    multi = list(_header_values(request, _API_KEYS_HEADER))
+    auths = list(_header_values(request, _AUTH_HEADER))
+    bearer: list[str] = []
+    for a in auths:
+        if isinstance(a, str) and a.lower().startswith("bearer "):
+            bearer.append(a.split(" ", 1)[1].strip())
+    candidates = _split_csv(single + multi + bearer)
+    return any(candidate in allowed for candidate in candidates)
+
+
+def require_api_key_dependency(request: Any) -> None:
+    """FastAPI dependency enforcing Task App authentication headers."""
+
+    allowed = allowed_environment_api_keys()
+    if not allowed:
+        raise http_exception(
+            503, "missing_environment_api_key", "ENVIRONMENT_API_KEY is not configured"
+        )
+    # Build candidate list for verbose diagnostics
+    single = list(_header_values(request, _API_KEY_HEADER))
+    multi = list(_header_values(request, _API_KEYS_HEADER))
+    auths = list(_header_values(request, _AUTH_HEADER))
+    bearer: list[str] = []
+    for a in auths:
+        if isinstance(a, str) and a.lower().startswith("bearer "):
+            bearer.append(a.split(" ", 1)[1].strip())
+    candidates = _split_csv(single + multi + bearer)
+    if not any(candidate in allowed for candidate in candidates):
+        with suppress(Exception):
+            print(
+                {
+                    "task_auth_failed": True,
+                    "allowed_first15": [k[:15] for k in allowed],
+                    "allowed_count": len(allowed),
+                    "got_first15": [c[:15] for c in candidates],
+                    "got_lens": [len(c) for c in candidates],
+                    "have_x_api_key": bool(single),
+                    "have_x_api_keys": bool(multi),
+                    "have_authorization": bool(auths),
+                },
+                flush=True,
+            )
+        # Use 400 to make failures unmistakable during preflight
+        raise http_exception(
+            400,
+            "unauthorised",
+            "API key missing or invalid",
+            extra={
+                "allowed_first15": [k[:15] for k in allowed],
+                "allowed_count": len(allowed),
+                "got_first15": [c[:15] for c in candidates],
+                "got_lens": [len(c) for c in candidates],
+            },
+        )

synth_ai/task/client.py

@@ -0,0 +1,167 @@
+"""Async HTTP client for interacting with Task Apps."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import Any
+
+import httpx
+from pydantic import BaseModel
+
+from .contracts import RolloutRequest, RolloutResponse, TaskInfo
+from .json import to_jsonable
+
+
+def _prepare_payload(payload: Any) -> Any:
+    if payload is None:
+        return None
+    if isinstance(payload, BaseModel):
+        return payload.model_dump(mode="json", by_alias=True)
+    return to_jsonable(payload)
+
+
+class TaskAppClient:
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str | None = None,
+        *,
+        timeout: float = 600.0,
+        retries: int = 3,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self.retries = max(1, retries)
+        self._client: httpx.AsyncClient | None = None
+        self.env = _TaskAppEnvironmentClient(self)
+
+    async def __aenter__(self) -> TaskAppClient:
+        await self._ensure_client()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.aclose()
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                base_url=self.base_url,
+                timeout=httpx.Timeout(self.timeout),
+                follow_redirects=True,
+            )
+        return self._client
+
+    def _headers(self) -> dict[str, str]:
+        headers: dict[str, str] = {}
+        # Primary key
+        primary = (self.api_key or "").strip()
+        if primary:
+            headers["X-API-Key"] = primary
+            # Also set Authorization for clients that read bearer tokens
+            headers.setdefault("Authorization", f"Bearer {primary}")
+        # Include ALL available environment keys via CSV in X-API-Keys
+        keys: list[str] = []
+        if primary:
+            keys.append(primary)
+        aliases = (os.getenv("ENVIRONMENT_API_KEY_ALIASES") or "").strip()
+        if aliases:
+            for part in aliases.split(","):
+                trimmed = part.strip()
+                if trimmed and trimmed not in keys:
+                    keys.append(trimmed)
+        if keys:
+            headers["X-API-Keys"] = ",".join(keys)
+        return headers
+
+    async def aclose(self) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    async def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, Any] | list[tuple[str, Any]] | None = None,
+        json_payload: Any = None,
+    ) -> httpx.Response:
+        client = await self._ensure_client()
+        payload = _prepare_payload(json_payload)
+        headers = self._headers()
+        last_exc: Exception | None = None
+        for attempt in range(self.retries):
+            try:
+                response = await client.request(
+                    method,
+                    path,
+                    headers=headers,
+                    params=params,
+                    json=payload,
+                )
+                response.raise_for_status()
+                return response
+            except httpx.HTTPStatusError as exc:
+                if 500 <= exc.response.status_code < 600 and attempt + 1 < self.retries:
+                    await asyncio.sleep(0.1 * (attempt + 1))
+                    last_exc = exc
+                    continue
+                raise
+            except httpx.HTTPError as exc:
+                last_exc = exc
+                if attempt + 1 >= self.retries:
+                    raise
+                await asyncio.sleep(0.1 * (attempt + 1))
+        if last_exc:  # pragma: no cover - defensive
+            raise last_exc
+        raise RuntimeError("Unreachable code in TaskAppClient._request")
+
+    async def health(self) -> dict[str, Any]:
+        response = await self._request("GET", "/health")
+        return response.json()
+
+    async def info(self) -> dict[str, Any]:
+        response = await self._request("GET", "/info")
+        return response.json()
+
+    async def task_info(self, seeds: list[int] | None = None) -> TaskInfo | list[TaskInfo]:
+        params: list[tuple[str, Any]] | None = None
+        if seeds:
+            params = [("seed", seed) for seed in seeds]
+        response = await self._request("GET", "/task_info", params=params)
+        data = response.json()
+        if isinstance(data, list):
+            return [TaskInfo.model_validate(item) for item in data]
+        return TaskInfo.model_validate(data)
+
+    async def rollout(self, request: RolloutRequest) -> RolloutResponse:
+        response = await self._request("POST", "/rollout", json_payload=request)
+        data = response.json()
+        return RolloutResponse.model_validate(data)
+
+
+class _TaskAppEnvironmentClient:
+    def __init__(self, client: TaskAppClient) -> None:
+        self._client = client
+
+    async def initialize(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
+        response = await self._client._request(
+            "POST", f"/env/{env_name}/initialize", json_payload=payload
+        )
+        return response.json()
+
+    async def step(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
+        response = await self._client._request(
+            "POST", f"/env/{env_name}/step", json_payload=payload
+        )
+        return response.json()
+
+    async def terminate(
+        self, env_name: str, payload: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
+        response = await self._client._request(
+            "POST", f"/env/{env_name}/terminate", json_payload=payload or {}
+        )
+        return response.json()

synth_ai/task/config.py

@@ -0,0 +1,261 @@
+"""Configuration dataclasses for task app CLI commands (eval, filter)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass(slots=True)
+class EvalConfig:
+    """Configuration for 'synth-ai eval' command.
+    
+    Validates and provides defaults for evaluation runs against task apps.
+    """
+    
+    # Required: Task app identifier
+    app_id: str
+    
+    # Required: Model to evaluate
+    model: str
+    
+    # Required: Seeds to run
+    seeds: list[int]
+    
+    # Optional: Task app URL (None = spawn in-process)
+    task_app_url: str | None = None
+    
+    # Optional: Data split to use
+    split: str = "train"
+    
+    # Optional: Maximum turns/steps per episode
+    max_turns: int | None = None
+    
+    # Optional: Maximum LLM calls per episode
+    max_llm_calls: int = 10
+    
+    # Optional: Concurrency for parallel rollouts
+    concurrency: int = 1
+    
+    # Optional: Environment name
+    env_name: str | None = None
+    
+    # Optional: Policy name
+    policy_name: str | None = None
+    
+    # Optional: Trace format ("compact", "full", "structured")
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+    
+    # Optional: Whether to return traces in response
+    return_trace: bool = False
+    
+    # Optional: Operations sequence (if not provided, generates default)
+    ops: list[str] | None = None
+    
+    # Optional: Environment config overrides
+    env_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Policy config overrides
+    policy_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for traces
+    metadata: dict[str, str] = field(default_factory=dict)
+    
+    # Optional: SQL query for metadata filtering
+    metadata_sql: str | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.app_id:
+            raise ValueError("app_id is required")
+        
+        if not self.model:
+            raise ValueError("model is required")
+        
+        if not self.seeds:
+            raise ValueError("seeds list cannot be empty")
+        
+        if not isinstance(self.seeds, list):
+            raise ValueError("seeds must be a list of integers")
+        
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        
+        if self.max_llm_calls < 1:
+            raise ValueError("max_llm_calls must be >= 1")
+        
+        if self.max_turns is not None and self.max_turns < 1:
+            raise ValueError("max_turns must be >= 1")
+        
+        if self.trace_format not in ("compact", "full", "structured"):
+            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
+        """Create EvalConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with eval configuration
+            
+        Returns:
+            Validated EvalConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "app_id": data.get("app_id"),
+            "model": data.get("model"),
+            "seeds": data.get("seeds", []),
+            "task_app_url": data.get("task_app_url"),
+            "split": data.get("split", "train"),
+            "max_turns": data.get("max_turns"),
+            "max_llm_calls": data.get("max_llm_calls", 10),
+            "concurrency": data.get("concurrency", 1),
+            "env_name": data.get("env_name"),
+            "policy_name": data.get("policy_name"),
+            "trace_format": data.get("trace_format", "compact"),
+            "return_trace": data.get("return_trace", False),
+            "ops": data.get("ops"),
+            "env_config": data.get("env_config", {}),
+            "policy_config": data.get("policy_config", {}),
+            "metadata": data.get("metadata", {}),
+            "metadata_sql": data.get("metadata_sql"),
+        }
+        
+        return cls(**config_dict)
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    """Configuration for 'synth-ai filter' command.
+    
+    Validates and provides defaults for filtering traces into SFT datasets.
+    """
+    
+    # Required: Database path or URL
+    db: str
+    
+    # Required: Output JSONL path
+    output: str
+    
+    # Optional: Filter by data splits
+    splits: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by task IDs
+    task_ids: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by models
+    models: list[str] = field(default_factory=list)
+    
+    # Optional: Minimum official score threshold
+    min_official_score: float | None = None
+    
+    # Optional: Maximum official score threshold
+    max_official_score: float | None = None
+    
+    # Optional: Minimum judge scores (judge_name -> min_score)
+    min_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Maximum judge scores (judge_name -> max_score)
+    max_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Limit number of examples
+    limit: int | None = None
+    
+    # Optional: Offset for pagination
+    offset: int | None = None
+    
+    # Optional: Whether to shuffle results
+    shuffle: bool = False
+    
+    # Optional: Random seed for shuffling
+    shuffle_seed: int | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.db:
+            raise ValueError("db (database path or URL) is required")
+        
+        if not self.output:
+            raise ValueError("output (JSONL file path) is required")
+        
+        # Validate output has .jsonl extension
+        output_path = Path(self.output)
+        if output_path.suffix.lower() not in (".jsonl", ".json"):
+            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
+        
+        # Validate score thresholds
+        if (
+            self.min_official_score is not None
+            and self.max_official_score is not None
+            and self.min_official_score > self.max_official_score
+        ):
+            raise ValueError("min_official_score cannot be greater than max_official_score")
+        
+        # Validate limit/offset
+        if self.limit is not None and self.limit < 1:
+            raise ValueError("limit must be >= 1")
+        
+        if self.offset is not None and self.offset < 0:
+            raise ValueError("offset must be >= 0")
+        
+        # Validate shuffle seed requires shuffle
+        if self.shuffle_seed is not None and not self.shuffle:
+            raise ValueError("shuffle_seed requires shuffle=true")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
+        """Create FilterConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with filter configuration
+            
+        Returns:
+            Validated FilterConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "db": data.get("db"),
+            "output": data.get("output"),
+            "splits": data.get("splits", []),
+            "task_ids": data.get("task_ids", []),
+            "models": data.get("models", []),
+            "min_official_score": data.get("min_official_score"),
+            "max_official_score": data.get("max_official_score"),
+            "min_judge_scores": data.get("min_judge_scores", {}),
+            "max_judge_scores": data.get("max_judge_scores", {}),
+            "limit": data.get("limit"),
+            "offset": data.get("offset"),
+            "shuffle": data.get("shuffle", False),
+            "shuffle_seed": data.get("shuffle_seed"),
+        }
+        
+        return cls(**config_dict)
+    
+    def get_db_url(self) -> str:
+        """Convert db path to proper SQLite URL if needed.
+        
+        Returns:
+            Database URL suitable for SQLAlchemy/aiosqlite
+        """
+        db_value = self.db.strip()
+        if "://" in db_value:
+            return db_value
+        else:
+            db_path = Path(db_value).expanduser().resolve()
+            # Ensure parent directory exists
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            return f"sqlite+aiosqlite:///{db_path}"
+    
+    def get_output_path(self) -> Path:
+        """Get resolved output path with parent directory created.
+        
+        Returns:
+            Resolved Path object with parent directory created
+        """
+        output_path = Path(self.output).expanduser().resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+
+