synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/task/auth.py

@@ -1,9 +1,11 @@
-from __future__ import annotations
-
 """Authentication helpers shared by Task Apps."""
 
+from __future__ import annotations
+
 import os
-from typing import Iterable, Optional, Any, Set
+from collections.abc import Iterable
+from contextlib import suppress
+from typing import Any
 
 from .errors import http_exception
 
@@ -12,7 +14,9 @@ _DEV_API_KEY_ENVS = ("dev_environment_api_key", "DEV_ENVIRONMENT_API_KEY")
 _API_KEY_HEADER = "x-api-key"
 _API_KEYS_HEADER = "x-api-keys"
 _AUTH_HEADER = "authorization"
-_API_KEY_ALIASES_ENV = "ENVIRONMENT_API_KEY_ALIASES"  # comma-separated list of additional valid keys
+_API_KEY_ALIASES_ENV = (
+    "ENVIRONMENT_API_KEY_ALIASES"  # comma-separated list of additional valid keys
+)
 
 
 def _mask(value: str, *, prefix: int = 4) -> str:
@@ -22,7 +26,7 @@ def _mask(value: str, *, prefix: int = 4) -> str:
     return f"{visible}{'…' if len(value) > prefix else ''}"
 
 
-def normalize_environment_api_key() -> Optional[str]:
+def normalize_environment_api_key() -> str | None:
     """Ensure `ENVIRONMENT_API_KEY` is populated from dev fallbacks.
 
     Returns the resolved key (if any) so callers can branch on configuration.
@@ -43,7 +47,7 @@ def normalize_environment_api_key() -> Optional[str]:
     return None
 
 
-def allowed_environment_api_keys() -> Set[str]:
+def allowed_environment_api_keys() -> set[str]:
     """Return the set of valid environment API keys for this Task App.
 
     Includes:
@@ -120,7 +124,9 @@ def require_api_key_dependency(request: Any) -> None:
 
     allowed = allowed_environment_api_keys()
     if not allowed:
-        raise http_exception(503, "missing_environment_api_key", "ENVIRONMENT_API_KEY is not configured")
+        raise http_exception(
+            503, "missing_environment_api_key", "ENVIRONMENT_API_KEY is not configured"
+        )
     # Build candidate list for verbose diagnostics
     single = list(_header_values(request, _API_KEY_HEADER))
     multi = list(_header_values(request, _API_KEYS_HEADER))
@@ -131,23 +137,29 @@ def require_api_key_dependency(request: Any) -> None:
             bearer.append(a.split(" ", 1)[1].strip())
     candidates = _split_csv(single + multi + bearer)
     if not any(candidate in allowed for candidate in candidates):
-        try:
-            print({
-                "task_auth_failed": True,
+        with suppress(Exception):
+            print(
+                {
+                    "task_auth_failed": True,
+                    "allowed_first15": [k[:15] for k in allowed],
+                    "allowed_count": len(allowed),
+                    "got_first15": [c[:15] for c in candidates],
+                    "got_lens": [len(c) for c in candidates],
+                    "have_x_api_key": bool(single),
+                    "have_x_api_keys": bool(multi),
+                    "have_authorization": bool(auths),
+                },
+                flush=True,
+            )
+        # Use 400 to make failures unmistakable during preflight
+        raise http_exception(
+            400,
+            "unauthorised",
+            "API key missing or invalid",
+            extra={
                 "allowed_first15": [k[:15] for k in allowed],
                 "allowed_count": len(allowed),
                 "got_first15": [c[:15] for c in candidates],
                 "got_lens": [len(c) for c in candidates],
-                "have_x_api_key": bool(single),
-                "have_x_api_keys": bool(multi),
-                "have_authorization": bool(auths),
-            }, flush=True)
-        except Exception:
-            pass
-        # Use 400 to make failures unmistakable during preflight
-        raise http_exception(400, "unauthorised", "API key missing or invalid", extra={
-            "allowed_first15": [k[:15] for k in allowed],
-            "allowed_count": len(allowed),
-            "got_first15": [c[:15] for c in candidates],
-            "got_lens": [len(c) for c in candidates],
-        })
+            },
+        )

synth_ai/task/client.py

@@ -1,10 +1,10 @@
-from __future__ import annotations
-
 """Async HTTP client for interacting with Task Apps."""
 
+from __future__ import annotations
+
 import asyncio
-from typing import Any, Dict, Iterable, List, Optional
 import os
+from typing import Any
 
 import httpx
 from pydantic import BaseModel
@@ -37,7 +37,7 @@ class TaskAppClient:
         self._client: httpx.AsyncClient | None = None
         self.env = _TaskAppEnvironmentClient(self)
 
-    async def __aenter__(self) -> "TaskAppClient":
+    async def __aenter__(self) -> TaskAppClient:
         await self._ensure_client()
         return self
 
@@ -53,8 +53,8 @@ class TaskAppClient:
             )
         return self._client
 
-    def _headers(self) -> Dict[str, str]:
-        headers: Dict[str, str] = {}
+    def _headers(self) -> dict[str, str]:
+        headers: dict[str, str] = {}
         # Primary key
         primary = (self.api_key or "").strip()
         if primary:
@@ -85,7 +85,7 @@ class TaskAppClient:
         method: str,
         path: str,
         *,
-        params: Optional[Dict[str, Any] | List[tuple[str, Any]]] = None,
+        params: dict[str, Any] | list[tuple[str, Any]] | None = None,
         json_payload: Any = None,
     ) -> httpx.Response:
         client = await self._ensure_client()
@@ -118,16 +118,16 @@ class TaskAppClient:
             raise last_exc
         raise RuntimeError("Unreachable code in TaskAppClient._request")
 
-    async def health(self) -> Dict[str, Any]:
+    async def health(self) -> dict[str, Any]:
         response = await self._request("GET", "/health")
         return response.json()
 
-    async def info(self) -> Dict[str, Any]:
+    async def info(self) -> dict[str, Any]:
         response = await self._request("GET", "/info")
         return response.json()
 
     async def task_info(self, seeds: list[int] | None = None) -> TaskInfo | list[TaskInfo]:
-        params: Optional[List[tuple[str, Any]]] = None
+        params: list[tuple[str, Any]] | None = None
         if seeds:
             params = [("seed", seed) for seed in seeds]
         response = await self._request("GET", "/task_info", params=params)
@@ -146,19 +146,21 @@ class _TaskAppEnvironmentClient:
     def __init__(self, client: TaskAppClient) -> None:
         self._client = client
 
-    async def initialize(self, env_name: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+    async def initialize(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
         response = await self._client._request(
             "POST", f"/env/{env_name}/initialize", json_payload=payload
         )
         return response.json()
 
-    async def step(self, env_name: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+    async def step(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
         response = await self._client._request(
             "POST", f"/env/{env_name}/step", json_payload=payload
         )
         return response.json()
 
-    async def terminate(self, env_name: str, payload: Dict[str, Any] | None = None) -> Dict[str, Any]:
+    async def terminate(
+        self, env_name: str, payload: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
         response = await self._client._request(
             "POST", f"/env/{env_name}/terminate", json_payload=payload or {}
         )

synth_ai/task/config.py

@@ -0,0 +1,261 @@
+"""Configuration dataclasses for task app CLI commands (eval, filter)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass(slots=True)
+class EvalConfig:
+    """Configuration for 'synth-ai eval' command.
+    
+    Validates and provides defaults for evaluation runs against task apps.
+    """
+    
+    # Required: Task app identifier
+    app_id: str
+    
+    # Required: Model to evaluate
+    model: str
+    
+    # Required: Seeds to run
+    seeds: list[int]
+    
+    # Optional: Task app URL (None = spawn in-process)
+    task_app_url: str | None = None
+    
+    # Optional: Data split to use
+    split: str = "train"
+    
+    # Optional: Maximum turns/steps per episode
+    max_turns: int | None = None
+    
+    # Optional: Maximum LLM calls per episode
+    max_llm_calls: int = 10
+    
+    # Optional: Concurrency for parallel rollouts
+    concurrency: int = 1
+    
+    # Optional: Environment name
+    env_name: str | None = None
+    
+    # Optional: Policy name
+    policy_name: str | None = None
+    
+    # Optional: Trace format ("compact", "full", "structured")
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+    
+    # Optional: Whether to return traces in response
+    return_trace: bool = False
+    
+    # Optional: Operations sequence (if not provided, generates default)
+    ops: list[str] | None = None
+    
+    # Optional: Environment config overrides
+    env_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Policy config overrides
+    policy_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for traces
+    metadata: dict[str, str] = field(default_factory=dict)
+    
+    # Optional: SQL query for metadata filtering
+    metadata_sql: str | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.app_id:
+            raise ValueError("app_id is required")
+        
+        if not self.model:
+            raise ValueError("model is required")
+        
+        if not self.seeds:
+            raise ValueError("seeds list cannot be empty")
+        
+        if not isinstance(self.seeds, list):
+            raise ValueError("seeds must be a list of integers")
+        
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        
+        if self.max_llm_calls < 1:
+            raise ValueError("max_llm_calls must be >= 1")
+        
+        if self.max_turns is not None and self.max_turns < 1:
+            raise ValueError("max_turns must be >= 1")
+        
+        if self.trace_format not in ("compact", "full", "structured"):
+            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
+        """Create EvalConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with eval configuration
+            
+        Returns:
+            Validated EvalConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "app_id": data.get("app_id"),
+            "model": data.get("model"),
+            "seeds": data.get("seeds", []),
+            "task_app_url": data.get("task_app_url"),
+            "split": data.get("split", "train"),
+            "max_turns": data.get("max_turns"),
+            "max_llm_calls": data.get("max_llm_calls", 10),
+            "concurrency": data.get("concurrency", 1),
+            "env_name": data.get("env_name"),
+            "policy_name": data.get("policy_name"),
+            "trace_format": data.get("trace_format", "compact"),
+            "return_trace": data.get("return_trace", False),
+            "ops": data.get("ops"),
+            "env_config": data.get("env_config", {}),
+            "policy_config": data.get("policy_config", {}),
+            "metadata": data.get("metadata", {}),
+            "metadata_sql": data.get("metadata_sql"),
+        }
+        
+        return cls(**config_dict)
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    """Configuration for 'synth-ai filter' command.
+    
+    Validates and provides defaults for filtering traces into SFT datasets.
+    """
+    
+    # Required: Database path or URL
+    db: str
+    
+    # Required: Output JSONL path
+    output: str
+    
+    # Optional: Filter by data splits
+    splits: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by task IDs
+    task_ids: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by models
+    models: list[str] = field(default_factory=list)
+    
+    # Optional: Minimum official score threshold
+    min_official_score: float | None = None
+    
+    # Optional: Maximum official score threshold
+    max_official_score: float | None = None
+    
+    # Optional: Minimum judge scores (judge_name -> min_score)
+    min_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Maximum judge scores (judge_name -> max_score)
+    max_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Limit number of examples
+    limit: int | None = None
+    
+    # Optional: Offset for pagination
+    offset: int | None = None
+    
+    # Optional: Whether to shuffle results
+    shuffle: bool = False
+    
+    # Optional: Random seed for shuffling
+    shuffle_seed: int | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.db:
+            raise ValueError("db (database path or URL) is required")
+        
+        if not self.output:
+            raise ValueError("output (JSONL file path) is required")
+        
+        # Validate output has .jsonl extension
+        output_path = Path(self.output)
+        if output_path.suffix.lower() not in (".jsonl", ".json"):
+            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
+        
+        # Validate score thresholds
+        if (
+            self.min_official_score is not None
+            and self.max_official_score is not None
+            and self.min_official_score > self.max_official_score
+        ):
+            raise ValueError("min_official_score cannot be greater than max_official_score")
+        
+        # Validate limit/offset
+        if self.limit is not None and self.limit < 1:
+            raise ValueError("limit must be >= 1")
+        
+        if self.offset is not None and self.offset < 0:
+            raise ValueError("offset must be >= 0")
+        
+        # Validate shuffle seed requires shuffle
+        if self.shuffle_seed is not None and not self.shuffle:
+            raise ValueError("shuffle_seed requires shuffle=true")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
+        """Create FilterConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with filter configuration
+            
+        Returns:
+            Validated FilterConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "db": data.get("db"),
+            "output": data.get("output"),
+            "splits": data.get("splits", []),
+            "task_ids": data.get("task_ids", []),
+            "models": data.get("models", []),
+            "min_official_score": data.get("min_official_score"),
+            "max_official_score": data.get("max_official_score"),
+            "min_judge_scores": data.get("min_judge_scores", {}),
+            "max_judge_scores": data.get("max_judge_scores", {}),
+            "limit": data.get("limit"),
+            "offset": data.get("offset"),
+            "shuffle": data.get("shuffle", False),
+            "shuffle_seed": data.get("shuffle_seed"),
+        }
+        
+        return cls(**config_dict)
+    
+    def get_db_url(self) -> str:
+        """Convert db path to proper SQLite URL if needed.
+        
+        Returns:
+            Database URL suitable for SQLAlchemy/aiosqlite
+        """
+        db_value = self.db.strip()
+        if "://" in db_value:
+            return db_value
+        else:
+            db_path = Path(db_value).expanduser().resolve()
+            # Ensure parent directory exists
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            return f"sqlite+aiosqlite:///{db_path}"
+    
+    def get_output_path(self) -> Path:
+        """Get resolved output path with parent directory created.
+        
+        Returns:
+            Resolved Path object with parent directory created
+        """
+        output_path = Path(self.output).expanduser().resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+
+