synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/task/rubrics/strict.py

@@ -0,0 +1,149 @@
+"""Strict rubric validators for step-wise judges.
+
+These validators enforce stricter constraints than the general-purpose rubrics:
+- Weights must be ≤ 1.0 and sum to exactly 1.0
+- Only weighted_sum aggregation is allowed
+- All required fields must be non-empty
+
+Used primarily for validation in judge configurations.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, Literal
+
+import pydantic
+
+
+class StrictCriterion(pydantic.BaseModel):
+    """Single scoring criterion with strict validation.
+    
+    Enforces:
+    - Weight ≤ 1.0 (for proper normalization)
+    - Weight > 0.0 (positive)
+    - Non-empty strings
+    """
+
+    id: str
+    description: str
+    weight: float
+    scale: str | None = None
+
+    @pydantic.field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("weight must be a finite number")
+        if value <= 0.0:
+            raise ValueError("weight must be positive")
+        if value > 1.0:
+            raise ValueError("weight must be <= 1.0")
+        return value
+
+    @pydantic.field_validator("id", "description", mode="before")
+    @classmethod
+    def _strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        return value
+
+
+class StrictRubric(pydantic.BaseModel):
+    """Strict rubric definition for step-wise judges.
+    
+    Enforces:
+    - Weights must sum to 1.0
+    - Only weighted_sum aggregation
+    - Non-empty version and goal_text
+    - At least one criterion
+    """
+
+    version: str
+    goal_text: str
+    aggregation: Literal["weighted_sum"]
+    criteria: list[StrictCriterion]
+
+    @pydantic.model_validator(mode="after")
+    def _validate_weights(self) -> StrictRubric:
+        if not self.criteria:
+            raise ValueError("rubric must declare at least one criterion")
+        total_weight = sum(criterion.weight for criterion in self.criteria)
+        if not math.isclose(total_weight, 1.0, abs_tol=1e-6, rel_tol=1e-6):
+            raise ValueError(
+                f"criterion weights must sum to 1 (got {total_weight:.6f})"
+            )
+        return self
+
+    @pydantic.field_validator("version")
+    @classmethod
+    def _non_empty_version(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("version string must not be empty")
+        return value
+
+    @pydantic.field_validator("goal_text")
+    @classmethod
+    def _non_empty_goal_text(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("goal_text must not be empty")
+        return value
+
+
+# Re-export pydantic's ValidationError for convenience
+ValidationError = pydantic.ValidationError
+
+
+def validate_rubric_dict(payload: dict[str, Any]) -> StrictRubric:
+    """Validate an in-memory rubric payload with strict rules.
+    
+    Args:
+        payload: Dictionary representing the rubric JSON
+        
+    Returns:
+        Validated StrictRubric instance
+        
+    Raises:
+        ValidationError: If payload is invalid or doesn't meet strict constraints
+    """
+    if not isinstance(payload, dict):
+        raise TypeError("rubric payload must be a dictionary")
+    return StrictRubric.model_validate(payload)
+
+
+def _load_payload_from_file(path: Path) -> dict[str, Any]:
+    """Load JSON rubric from file."""
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Unsupported rubric file type: {path}")
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text)
+
+
+def validate_rubric_file(path: Path) -> StrictRubric:
+    """Load and validate a rubric file with strict rules.
+    
+    Args:
+        path: Path to a JSON rubric document
+        
+    Returns:
+        Validated StrictRubric instance
+    """
+    payload = _load_payload_from_file(path)
+    return validate_rubric_dict(payload)
+
+
+def validate_rubric_files(paths: Iterable[Path]) -> list[StrictRubric]:
+    """Validate multiple rubric files with strict rules.
+    
+    Useful for bulk validation inside tests or CI checks.
+    """
+    validated: list[StrictRubric] = []
+    for path in paths:
+        validated.append(validate_rubric_file(path))
+    return validated
+

synth_ai/task/rubrics.py

@@ -1,10 +1,11 @@
-from __future__ import annotations
-
 """Rubric schema, loading, and scoring helpers for Task Apps."""
 
+from __future__ import annotations
+
 import json
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Any, Dict, Iterable, Optional
+from typing import Any
 
 from pydantic import BaseModel, Field, field_validator
 
@@ -48,14 +49,14 @@ class Rubric(BaseModel):
         return criteria
 
 
-def _load_text(source: str) -> tuple[str, Optional[str]]:
+def _load_text(source: str) -> tuple[str, str | None]:
     path = Path(source)
     if path.exists():
         return path.read_text(encoding="utf-8"), path.suffix.lower()
     return source, None
 
 
-def _parse_structured(text: str, suffix: Optional[str]) -> Dict[str, Any]:
+def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
     text = text.strip()
     if not text:
         raise ValueError("Rubric source is empty")
@@ -66,7 +67,7 @@ def _parse_structured(text: str, suffix: Optional[str]) -> Dict[str, Any]:
             raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
         data = yaml.safe_load(text)
         if not isinstance(data, dict):
-            raise ValueError("Rubric YAML must produce a mapping")
+            raise ValueError("Rubric YAML must produce a mapping") from None
         return data
     if text.startswith("{"):
         return json.loads(text)
@@ -85,7 +86,7 @@ def _parse_structured(text: str, suffix: Optional[str]) -> Dict[str, Any]:
             raise RuntimeError("PyYAML is required to load rubric text") from exc
         data = yaml.safe_load(text)
         if not isinstance(data, dict):
-            raise ValueError("Rubric text must decode to a mapping")
+            raise ValueError("Rubric text must decode to a mapping") from None
         return data
 
 
@@ -148,17 +149,19 @@ def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None
     )
 
 
-def _as_float(value: Any) -> Optional[float]:
+def _as_float(value: Any) -> float | None:
     try:
         return float(value)
     except Exception:
         return None
 
 
-def _score(criteria: Iterable[Criterion], values: Dict[str, float], aggregation: str) -> Dict[str, Any]:
+def _score(
+    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
+) -> dict[str, Any]:
     if aggregation == "inherit":
         aggregation = "weighted_sum"
-    per_criterion: Dict[str, Dict[str, Any]] = {}
+    per_criterion: dict[str, dict[str, Any]] = {}
     total = 0.0
     total_weight = 0.0
     for criterion in criteria:
@@ -184,10 +187,12 @@ def _score(criteria: Iterable[Criterion], values: Dict[str, float], aggregation:
     }
 
 
-def score_events_against_rubric(events: list[dict[str, Any]], rubric: Rubric | None) -> Dict[str, Any]:
+def score_events_against_rubric(
+    events: list[dict[str, Any]], rubric: Rubric | None
+) -> dict[str, Any]:
     if rubric is None:
         return {"aggregation": "none", "score": None, "per_criterion": {}}
-    values: Dict[str, float] = {}
+    values: dict[str, float] = {}
     for event in events or []:
         if not isinstance(event, dict):
             continue
@@ -198,12 +203,14 @@ def score_events_against_rubric(events: list[dict[str, Any]], rubric: Rubric | N
     return _score(rubric.criteria, values, rubric.aggregation)
 
 
-def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> Dict[str, Any]:
+def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
     if rubric is None:
         return {"aggregation": "none", "score": None, "per_criterion": {}}
-    values: Dict[str, float] = {}
+    values: dict[str, float] = {}
     if isinstance(outcome, dict):
-        candidates = outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        candidates = (
+            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        )
         if isinstance(candidates, dict):
             for key, value in candidates.items():
                 score = _as_float(value)

synth_ai/task/server.py

@@ -1,44 +1,44 @@
-from __future__ import annotations
-
 """FastAPI scaffolding for Task Apps (local dev + deployment)."""
 
+from __future__ import annotations
+
 import asyncio
 import inspect
 import os
+from collections.abc import Awaitable, Callable, Iterable, Mapping, MutableMapping, Sequence
+from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Awaitable, Callable, Iterable, Mapping, MutableMapping, Sequence
+from typing import Any
 
 import httpx
 from fastapi import APIRouter, Depends, FastAPI, Query, Request
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse
 from starlette.middleware import Middleware
 
-from .auth import (
-    is_api_key_header_authorized,
-    normalize_environment_api_key,
-    require_api_key_dependency,
-)
+from .auth import normalize_environment_api_key, require_api_key_dependency
 from .contracts import RolloutRequest, RolloutResponse, TaskInfo
 from .datasets import TaskDatasetRegistry
 from .errors import http_exception
 from .json import to_jsonable
 from .proxy import (
+    inject_system_hint,
     prepare_for_groq,
     prepare_for_openai,
-    inject_system_hint,
     synthesize_tool_call_if_missing,
 )
 from .rubrics import Rubric
 from .vendors import get_groq_key_or_503, get_openai_key_or_503, normalize_vendor_keys
 
-
 TasksetDescriptor = Callable[[], Mapping[str, Any] | Awaitable[Mapping[str, Any]]]
 InstanceProvider = Callable[[Sequence[int]], Iterable[TaskInfo] | Awaitable[Iterable[TaskInfo]]]
 RolloutExecutor = Callable[[RolloutRequest, Request], Any | Awaitable[Any]]
 
 
+def _default_app_state() -> dict[str, Any]:
+    return {}
+
+
 @dataclass(slots=True)
 class RubricBundle:
     """Optional rubrics advertised by the task app."""
@@ -70,18 +70,18 @@ class TaskAppConfig:
     provide_task_instances: InstanceProvider
     rollout: RolloutExecutor
     dataset_registry: TaskDatasetRegistry | None = None
-    rubrics: RubricBundle = field(default_factory=RubricBundle)
+    rubrics: RubricBundle | None = field(default_factory=RubricBundle)
     proxy: ProxyConfig | None = None
     routers: Sequence[APIRouter] = field(default_factory=tuple)
     middleware: Sequence[Middleware] = field(default_factory=tuple)
-    app_state: Mapping[str, Any] = field(default_factory=dict)
+    app_state: MutableMapping[str, Any] = field(default_factory=_default_app_state)
     require_api_key: bool = True
     expose_debug_env: bool = True
     cors_origins: Sequence[str] | None = None
     startup_hooks: Sequence[Callable[[], None | Awaitable[None]]] = field(default_factory=tuple)
     shutdown_hooks: Sequence[Callable[[], None | Awaitable[None]]] = field(default_factory=tuple)
 
-    def clone(self) -> "TaskAppConfig":
+    def clone(self) -> TaskAppConfig:
         """Return a shallow copy safe to mutate when wiring the app."""
 
         return TaskAppConfig(
@@ -93,7 +93,7 @@ class TaskAppConfig:
             provide_task_instances=self.provide_task_instances,
             rollout=self.rollout,
             dataset_registry=self.dataset_registry,
-            rubrics=self.rubrics,
+            rubrics=self.rubrics or RubricBundle(),
             proxy=self.proxy,
             routers=tuple(self.routers),
             middleware=tuple(self.middleware),
@@ -120,7 +120,9 @@ def _ensure_task_info(obj: Any) -> TaskInfo:
         return obj
     if isinstance(obj, MutableMapping):
         return TaskInfo.model_validate(obj)
-    raise TypeError(f"Task instance provider must yield TaskInfo-compatible objects (got {type(obj)!r})")
+    raise TypeError(
+        f"Task instance provider must yield TaskInfo-compatible objects (got {type(obj)!r})"
+    )
 
 
 def _normalise_seeds(values: Sequence[int]) -> list[int]:
@@ -140,7 +142,9 @@ def _build_proxy_routes(
     if not proxy:
         return
 
-    async def _call_vendor(url: str, payload: dict[str, Any], headers: dict[str, str]) -> dict[str, Any]:
+    async def _call_vendor(
+        url: str, payload: dict[str, Any], headers: dict[str, str]
+    ) -> dict[str, Any]:
         async with httpx.AsyncClient(timeout=httpx.Timeout(600.0), follow_redirects=True) as client:
             response = await client.post(url, json=payload, headers=headers)
         data = (
@@ -168,13 +172,17 @@ def _build_proxy_routes(
             msg_count = len(messages) if isinstance(messages, list) else 0
             tool_count = len(payload.get("tools") or []) if isinstance(payload, dict) else 0
             model = payload.get("model") if isinstance(payload, dict) else None
-            print(f"[task:proxy:{route}] model={model} messages={msg_count} tools={tool_count}", flush=True)
+            print(
+                f"[task:proxy:{route}] model={model} messages={msg_count} tools={tool_count}",
+                flush=True,
+            )
         except Exception:  # pragma: no cover - best effort logging
             pass
 
     system_hint = proxy.system_hint
 
     if proxy.enable_openai:
+
         @app.post("/proxy/v1/chat/completions", dependencies=[Depends(auth_dependency)])
         async def proxy_openai(body: dict[str, Any], request: Request) -> Any:  # type: ignore[no-redef]
             key = get_openai_key_or_503()
@@ -187,6 +195,7 @@ def _build_proxy_routes(
             return to_jsonable(sanitized)
 
     if proxy.enable_groq:
+
         @app.post("/proxy/groq/v1/chat/completions", dependencies=[Depends(auth_dependency)])
         async def proxy_groq(body: dict[str, Any], request: Request) -> Any:  # type: ignore[no-redef]
             key = get_groq_key_or_503()
@@ -194,7 +203,9 @@ def _build_proxy_routes(
             payload = prepare_for_groq(model, body)
             payload = inject_system_hint(payload, system_hint or "")
             _log_proxy("groq", payload)
-            data = await _call_vendor(proxy.groq_url.rstrip("/"), payload, {"Authorization": f"Bearer {key}"})
+            data = await _call_vendor(
+                proxy.groq_url.rstrip("/"), payload, {"Authorization": f"Bearer {key}"}
+            )
             sanitized = synthesize_tool_call_if_missing(data)
             return to_jsonable(sanitized)
 
@@ -210,6 +221,7 @@ def _auth_dependency_factory(config: TaskAppConfig) -> Callable[[Request], None]
 
 def create_task_app(config: TaskAppConfig) -> FastAPI:
     cfg = config.clone()
+    cfg.rubrics = cfg.rubrics or RubricBundle()
     app = FastAPI(title=cfg.name, description=cfg.description)
 
     for key, value in cfg.app_state.items():
@@ -254,17 +266,19 @@ def create_task_app(config: TaskAppConfig) -> FastAPI:
             return _maybe_await(hook(app))  # type: ignore[misc]
         return _maybe_await(hook())
 
-    @app.on_event("startup")
-    async def _startup() -> None:  # pragma: no cover - FastAPI lifecycle
+    @asynccontextmanager
+    async def lifespan(_: FastAPI):
         normalize_environment_api_key()
         normalize_vendor_keys()
         for hook in cfg.startup_hooks:
             await _call_hook(hook)
+        try:
+            yield
+        finally:
+            for hook in cfg.shutdown_hooks:
+                await _call_hook(hook)
 
-    @app.on_event("shutdown")
-    async def _shutdown() -> None:  # pragma: no cover - FastAPI lifecycle
-        for hook in cfg.shutdown_hooks:
-            await _call_hook(hook)
+    app.router.lifespan_context = lifespan
 
     @app.get("/")
     async def root() -> Mapping[str, Any]:
@@ -278,26 +292,39 @@ def create_task_app(config: TaskAppConfig) -> FastAPI:
     async def health(request: Request) -> Mapping[str, Any]:
         # If we got here, auth_dependency already verified the key exactly matches
         expected = normalize_environment_api_key()
-        return to_jsonable({"healthy": True, "auth": {"required": True, "expected_prefix": (expected[:6] + '...') if expected else '<unset>'}})
+        return to_jsonable(
+            {
+                "healthy": True,
+                "auth": {
+                    "required": True,
+                    "expected_prefix": (expected[:6] + "...") if expected else "<unset>",
+                },
+            }
+        )
+
+    @app.post("/done", dependencies=[Depends(auth_dependency)])
+    async def done() -> Mapping[str, Any]:
+        # Coordination endpoint for tests and automation; indicates app is reachable
+        return to_jsonable({"ok": True, "service": cfg.app_id})
 
     @app.get("/info", dependencies=[Depends(auth_dependency)])
     async def info() -> Mapping[str, Any]:
         dataset_meta = cfg.base_task_info.dataset
         rubrics: dict[str, Any] | None = None
-        if cfg.rubrics.outcome or cfg.rubrics.events:
+        rubric_bundle = cfg.rubrics
+        if rubric_bundle and (rubric_bundle.outcome or rubric_bundle.events):
             rubrics = {
-                "outcome": cfg.rubrics.outcome.model_dump() if cfg.rubrics.outcome else None,
-                "events": cfg.rubrics.events.model_dump() if cfg.rubrics.events else None,
+                "outcome": rubric_bundle.outcome.model_dump() if rubric_bundle.outcome else None,
+                "events": rubric_bundle.events.model_dump() if rubric_bundle.events else None,
             }
         payload = {
             "service": {
                 "task": cfg.base_task_info.task,
-                "version": cfg.base_task_info.task.get("version"),
+                "version": cfg.base_task_info.task.version,
             },
             "dataset": dataset_meta,
             "rubrics": rubrics,
             "inference": cfg.base_task_info.inference,
-            "capabilities": cfg.base_task_info.capabilities,
             "limits": cfg.base_task_info.limits,
         }
         return to_jsonable(payload)
@@ -335,6 +362,7 @@ def create_task_app(config: TaskAppConfig) -> FastAPI:
         raise TypeError("Rollout executor must return RolloutResponse or mapping")
 
     if cfg.expose_debug_env:
+
         @app.get("/debug/env", dependencies=[Depends(auth_dependency)])
         async def debug_env() -> Mapping[str, Any]:
             def _mask(value: str | None) -> str:
@@ -387,6 +415,12 @@ def run_task_app(
         print(f"[task:server] Loaded environment from: {', '.join(loaded_files)}", flush=True)
 
     config = config_factory()
+    # Defensive: ensure the factory produced a valid TaskAppConfig to avoid
+    # confusing attribute errors later in the boot sequence.
+    if not isinstance(config, TaskAppConfig):  # type: ignore[arg-type]
+        raise TypeError(
+            f"Task app config_factory must return TaskAppConfig, got {type(config).__name__}"
+        )
     app = create_task_app(config)
 
     try: