synth-ai 0.2.12__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

synth_ai/environments/examples/sokoban/taskset.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 import logging
 import os
+from collections.abc import Iterable, Sequence
 from dataclasses import asdict, dataclass, fields
 from typing import List, Tuple
 from uuid import UUID, uuid4
@@ -18,6 +21,7 @@ from synth_ai.environments.tasks.core import (
     TaskInstanceMetadataFilter,
     TaskInstanceSet,
 )
+from synth_ai.task.contracts import TaskInfo
 
 logger = logging.getLogger(__name__)
 
@@ -96,6 +100,118 @@ class SokobanTaskInstance(TaskInstance):
         return cls(**filtered_data)
 
 
+def _base_task_info_template() -> TaskInfo:
+    return TaskInfo(
+        task={"id": "sokoban", "name": "Sokoban", "version": "1.0.0"},
+        environment="sokoban",
+        action_space={
+            "type": "tool_call",
+            "tools": [{"name": "interact", "schema": {"action": "int"}}],
+            "max_calls": 1,
+        },
+        observation={"summary": "Sokoban grid observation", "keys": ["grid", "player"]},
+        dataset={"id": "sokoban", "name": "Sokoban", "version": "1.0.0"},
+        rubric={"version": "1", "criteria_count": 1, "source": "inline"},
+        inference={"supports_proxy": False},
+        capabilities={"supports_rollout": True, "supports_env_lifecycle": True},
+        limits={"max_turns": 200},
+    )
+
+
+class SokobanTaskSet:
+    """Minimal helper compatible with Task App expectations."""
+
+    def __init__(self) -> None:
+        self._taskset: TaskInstanceSet | None = None
+        self._seed_index: dict[int, SokobanTaskInstance] = {}
+        self._base_info = _base_task_info_template()
+
+    async def _ensure_loaded(self) -> TaskInstanceSet:
+        if self._taskset is None:
+            dataset = await create_sokoban_taskset()
+            self._taskset = dataset
+            self._seed_index.clear()
+            for inst in dataset.instances:
+                try:
+                    seed_value = int(getattr(inst.metadata, "seed"))
+                except Exception:
+                    continue
+                # Keep the first instance encountered for a seed
+                self._seed_index.setdefault(seed_value, inst)
+        return self._taskset
+
+    def describe(self) -> dict[str, object]:
+        if not self._taskset:
+            return {"id": "sokoban", "name": "Sokoban"}
+        return {
+            "id": "sokoban",
+            "name": self._taskset.name,
+            "description": self._taskset.description,
+            "instance_count": len(self._taskset.instances),
+        }
+
+    async def provide_task_instances(self, seeds: Sequence[int]) -> Iterable[TaskInfo]:
+        await self._ensure_loaded()
+        if not seeds:
+            return []
+
+        infos: list[TaskInfo] = []
+        for raw_seed in seeds:
+            try:
+                seed_value = int(raw_seed)
+            except Exception:
+                continue
+
+            instance = self._seed_index.get(seed_value)
+            if instance is None:
+                # Attempt to construct on the fly; try configured difficulties in order
+                for difficulty in DIFFICULTY_CONFIGS:
+                    try:
+                        instance = await create_task_instance_from_seed(difficulty, seed_value)
+                        break
+                    except Exception:
+                        continue
+                if instance is None:
+                    continue
+                self._seed_index[seed_value] = instance
+
+            metadata = getattr(instance, "metadata", None)
+            base_info = self._base_info.model_copy(deep=True)
+
+            observation = dict(base_info.observation)
+            dataset_info = dict(base_info.dataset)
+            task_metadata = {"seed": seed_value}
+
+            if metadata is not None:
+                for key in ("difficulty", "num_boxes", "dim_room", "max_steps", "shortest_path_length"):
+                    value = getattr(metadata, key, None)
+                    if value is not None:
+                        observation[key] = value
+                        task_metadata[key] = value
+                dataset_info.update(
+                    {
+                        "seed": getattr(metadata, "seed", seed_value),
+                        "difficulty": getattr(metadata, "difficulty", None),
+                        "num_boxes": getattr(metadata, "num_boxes", None),
+                        "dim_room": getattr(metadata, "dim_room", None),
+                    }
+                )
+                generation_params = getattr(metadata, "generation_params", None)
+                if generation_params is not None:
+                    task_metadata["generation_params"] = generation_params
+
+            infos.append(
+                base_info.model_copy(
+                    update={
+                        "observation": observation,
+                        "dataset": dataset_info,
+                        "task_metadata": task_metadata,
+                    }
+                )
+            )
+        return infos
+
+
 async def create_sokoban_taskset() -> TaskInstanceSet:
     """Generates Sokoban task instances from pre-generated verified puzzles."""
     instances = []

synth_ai/environments/examples/verilog/engine.py

@@ -67,6 +67,16 @@ class VerilogStepPenaltyComponent(RewardComponent):
         return self.penalty
 
 
+class VerilogSubmitSuccessComponent(RewardComponent):
+    """Reward for successful submission (tests passed)."""
+    async def score(self, state: VerilogPublicState, action: Any) -> float:
+        if hasattr(action, "get") and action.get("type") == "submit":
+            # Check if submission passed
+            if action.get("passed", False):
+                return 10.0  # Large reward for completing the task correctly
+        return 0.0
+
+
 class VerilogEngine(StatefulEngine):
     """
     Stateful Verilog evaluation engine with persistent artifact snapshots.
@@ -81,6 +91,7 @@ class VerilogEngine(StatefulEngine):
             components=[
                 VerilogCompileSuccessComponent(),
                 VerilogSimulationPassComponent(),
+                VerilogSubmitSuccessComponent(),
                 VerilogStepPenaltyComponent(penalty=-0.01),
             ]
         )
@@ -284,13 +295,28 @@ class VerilogEngine(StatefulEngine):
 
     async def submit(self) -> Dict[str, Any]:
         """Submit solution for grading."""
-        # For now, simple check based on last simulation
-        # In a full implementation, this would call the task's verify method
+        # Check if the last simulation passed
+        # Parse the last simulation output to determine if tests passed
+        passed = False
+        detail = "No simulation run yet"
+        
+        if self._last_simulate_output:
+            stdout = self._last_simulate_output
+            passed = (
+                "ALL_TESTS_PASSED" in stdout
+                or ("Mismatches: 0 " in stdout and "samples" in stdout)
+                or ("no mismatches" in stdout.lower() and "errors" not in stdout.lower())
+            )
+            if passed:
+                detail = "All tests passed"
+            else:
+                detail = "Tests failed - please review simulation output"
+        
         return {
             "ok": True,
             "type": "submit",
-            "passed": True,  # Placeholder
-            "detail": "Submission processed",
+            "passed": passed,
+            "detail": detail,
             "submitted": True,
         }
 

synth_ai/evals/__init__.py

@@ -0,0 +1,15 @@
+from .client import JudgeClient, JudgeOptions, JudgeScoreResponse
+from .types import Judgement, RewardJudgement, RewardMetadata, Track, TrackAggregate
+
+__all__ = [
+	"JudgeClient",
+	"JudgeOptions",
+	"JudgeScoreResponse",
+	"Judgement",
+	"RewardJudgement",
+	"RewardMetadata",
+	"Track",
+	"TrackAggregate",
+]
+
+

synth_ai/evals/client.py

@@ -0,0 +1,82 @@
+"""Experimental Judge API client.
+
+This surface is experimental and subject to change without notice.
+Set environment variable `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from typing import Any, Literal, TypedDict
+
+from synth_ai.http import AsyncHttpClient, HTTPError
+from synth_ai.tracing_v3.serialization import normalize_for_json
+
+Provider = Literal["groq", "gemini"]
+
+
+class JudgeOptions(TypedDict, total=False):
+    event: bool
+    outcome: bool
+    rubric_id: str
+    rubric_overrides: dict[str, Any]
+    provider: Provider
+    model: str
+    max_concurrency: int
+
+
+class JudgeScoreResponse(TypedDict, total=False):
+    status: str
+    event_rewards: list[dict[str, Any]]
+    outcome_reward: dict[str, Any]
+    details: dict[str, Any]
+
+
+class JudgeClient:
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None:
+        _silence = (os.getenv("SYNTH_SILENCE_EXPERIMENTAL") or "").strip().lower()
+        if _silence not in {"1", "true", "t", "yes", "y", "on"}:
+            warnings.warn(
+                "Experimental API: synth_ai.evals.JudgeClient is experimental and may change without notice.",
+                UserWarning,
+                stacklevel=2,
+            )
+        self._base = base_url.rstrip("/")
+        self._key = api_key
+        self._timeout = timeout
+
+    async def score(
+        self,
+        *,
+        trace: dict[str, Any] | Any,
+        policy_name: str,
+        task_app_id: str,
+        options: JudgeOptions,
+        task_app_base_url: str | None = None,
+    ) -> JudgeScoreResponse:
+        body = {
+            "policy_name": policy_name,
+            "task_app": {"id": task_app_id, **({"base_url": task_app_base_url} if task_app_base_url else {})},
+            "trace": normalize_for_json(trace),
+            "options": options or {},
+        }
+        try:
+            async with AsyncHttpClient(self._base, self._key, timeout=self._timeout) as http:
+                js = await http.post_json("/api/judge/v1/score", json=body)
+                if not isinstance(js, dict):
+                    raise ValueError("invalid_judge_response_shape")
+                return js  # type: ignore[return-value]
+        except HTTPError as err:  # map to friendlier exceptions
+            status = int(getattr(err, "status", 0) or 0)
+            if status in (400, 422):
+                raise ValueError(f"judge_validation_error: {err.detail}") from err
+            if status in (401, 403):
+                raise PermissionError(f"judge_auth_error: {err.detail}") from err
+            if status == 404:
+                raise FileNotFoundError(f"judge_route_not_found: {err.detail}") from err
+            if status == 429:
+                raise Exception("judge_rate_limited") from err  # replace with RetryLater in future
+            if status >= 500:
+                raise Exception("judge_transient_error") from err  # replace with TransientError in future
+            raise

synth_ai/evals/types.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Literal, TypedDict
+
+Track = Literal["process", "reasoning", "progress", "outcome"]
+
+
+class Judgement(TypedDict, total=False):
+	key: str
+	title: str
+	description: str
+	score: float
+	reason: str
+	confidence: float
+	scale: Literal["binary", "bounded", "count", "custom"]
+	source: dict
+
+
+class RewardJudgement(TypedDict, total=False):
+	judgement: Judgement
+	scope: Literal["step", "event", "outcome"]
+	turn: int | None
+	episode_id: str | None
+	reward_value: float | None
+	links: dict
+
+
+class TrackAggregate(TypedDict, total=False):
+	mean: float
+	median: float
+	std: float
+	n: int
+
+
+class RewardMetadata(TypedDict, total=False):
+	per_window: list[RewardJudgement]
+	aggregates: dict[Track, TrackAggregate]
+	overall: dict[str, float]  # {"final_outcome_score": float}
+	rubric: dict               # {"ids": {...}, "hash": "..."}
+	model_info: dict           # {"model": "...", ...}
+
+

synth_ai/jobs/client.py

@@ -1,20 +1,32 @@
 from __future__ import annotations
 
 import importlib
-from typing import Any
+from collections.abc import Callable
+from typing import Any, cast
 
 try:
-    normalize_model_identifier = importlib.import_module("synth_ai.api.models.supported").normalize_model_identifier
+    _supported_module = cast(
+        Any, importlib.import_module("synth_ai.api.models.supported")
+    )
+    normalize_model_identifier = cast(
+        Callable[[str], str], _supported_module.normalize_model_identifier
+    )
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load supported model utilities") from exc
 
 try:
-    AsyncHttpClient = importlib.import_module("synth_ai.http").AsyncHttpClient
+    _http_module = cast(Any, importlib.import_module("synth_ai.http"))
+    AsyncHttpClient = _http_module.AsyncHttpClient
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load HTTP client") from exc
 
 try:
-    prepare_sft_job_payload = importlib.import_module("synth_ai.learning.sft.config").prepare_sft_job_payload
+    _sft_config_module = cast(
+        Any, importlib.import_module("synth_ai.learning.sft.config")
+    )
+    prepare_sft_job_payload = cast(
+        Callable[..., dict[str, Any]], _sft_config_module.prepare_sft_job_payload
+    )
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load SFT configuration helpers") from exc
 

synth_ai/judge_schemas.py

@@ -0,0 +1,127 @@
+"""
+Judge API Contract Schemas
+
+These schemas define the expected structure for requests and responses
+to the judge scoring endpoint at POST /api/judge/v1/score.
+
+This is the canonical contract that the backend MUST conform to.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class CriterionScorePayload(BaseModel):
+    """Per-criterion score returned by the judge."""
+
+    score: float = Field(..., description="Numeric score for this criterion")
+    reason: str = Field(default="", description="Explanation for the score")
+    weight: float = Field(default=1.0, description="Weight of this criterion")
+    description: str = Field(default="", description="Description of the criterion")
+
+
+class ReviewPayload(BaseModel):
+    """Rubric review (event-level or outcome-level)."""
+
+    criteria: dict[str, CriterionScorePayload] = Field(
+        default_factory=dict,
+        description="Map of criterion keys to their scores"
+    )
+    total: float = Field(default=0.0, description="Aggregated total score")
+    summary: str | None = Field(None, description="Optional text summary")
+
+
+class JudgeScoreResponse(BaseModel):
+    """
+    Response body for POST /api/judge/v1/score.
+    
+    This is the canonical contract that judge backends MUST return.
+    """
+
+    status: Literal["ok", "failed"] = Field(default="ok", description="Request status")
+    event_reviews: list[ReviewPayload] = Field(
+        default_factory=list,
+        description="List of per-event rubric reviews (one per step)"
+    )
+    outcome_review: ReviewPayload | None = Field(
+        None,
+        description="Optional outcome-level rubric review"
+    )
+    event_totals: list[float] = Field(
+        default_factory=list,
+        description="List of aggregated scores per event (matches event_reviews length)"
+    )
+    details: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional details (provider, latency, etc.)"
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Request metadata (provider, options, etc.)"
+    )
+
+    def aggregate_event_reward(self) -> float | None:
+        """
+        Aggregate all event totals into a single reward.
+        
+        Returns:
+            Sum of all event_totals, or None if empty
+        """
+        if not self.event_totals:
+            return None
+        return sum(self.event_totals)
+
+    def aggregate_outcome_reward(self) -> float | None:
+        """
+        Extract outcome reward from outcome_review.
+        
+        Returns:
+            outcome_review.total, or None if no outcome review
+        """
+        if self.outcome_review is None:
+            return None
+        return self.outcome_review.total
+
+
+# Request schemas for completeness
+
+class JudgeTaskApp(BaseModel):
+    """Task application metadata."""
+    
+    id: str = Field(..., description="Task app identifier")
+    base_url: str | None = Field(None, description="Optional base URL for task app")
+
+
+class JudgeOptions(BaseModel):
+    """Judge provider and configuration options."""
+    
+    provider: str | None = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: str | None = Field(None, description="Model identifier")
+    rubric_id: str | None = Field(None, description="Rubric identifier")
+    event: bool = Field(True, description="Enable event-level judging")
+    outcome: bool = Field(True, description="Enable outcome-level judging")
+
+
+class JudgeTracePayload(BaseModel):
+    """Trace payload containing trajectory context."""
+    
+    event_history: list[dict[str, Any]] = Field(..., description="List of events/steps")
+    markov_blanket_message_history: list[dict[str, Any]] = Field(
+        default_factory=list,
+        description="Optional message history for context"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
+
+
+class JudgeScoreRequest(BaseModel):
+    """Request body for POST /api/judge/v1/score."""
+    
+    policy_name: str = Field(..., description="Name of the policy being evaluated")
+    task_app: JudgeTaskApp = Field(..., description="Task application metadata")
+    trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
+    options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
+    rubric: dict[str, Any] | None = Field(None, description="Optional explicit rubric criteria")
+

synth_ai/task/__init__.py

@@ -5,6 +5,9 @@ from .auth import (
 )
 from .client import TaskAppClient
 from .contracts import (
+    DatasetInfo,
+    InferenceInfo,
+    LimitsInfo,
     RolloutEnvSpec,
     RolloutMetrics,
     RolloutPolicySpec,
@@ -14,8 +17,10 @@ from .contracts import (
     RolloutSafetyConfig,
     RolloutStep,
     RolloutTrajectory,
-    TaskAppContract,
+    RubricInfo,
+    RubricSection,
     TaskAppEndpoints,
+    TaskDescriptor,
     TaskInfo,
 )
 from .datasets import TaskDatasetRegistry, TaskDatasetSpec
@@ -23,7 +28,6 @@ from .errors import error_payload, http_exception, json_error_response
 from .health import task_app_health
 from .json import to_jsonable
 from .proxy import (
-    INTERACT_TOOL_SCHEMA,
     extract_message_text,
     inject_system_hint,
     parse_tool_call_from_text,
@@ -46,7 +50,7 @@ from .server import (
     create_task_app,
     run_task_app,
 )
-from .validators import validate_task_app_url
+from .validators import validate_task_app_endpoint, validate_task_app_url
 from .vendors import (
     get_groq_key_or_503,
     get_openai_key_or_503,
@@ -55,8 +59,8 @@ from .vendors import (
 
 __all__ = [
     "validate_task_app_url",
+    "validate_task_app_endpoint",
     "task_app_health",
-    "TaskAppContract",
     "TaskAppEndpoints",
     "RolloutEnvSpec",
     "RolloutPolicySpec",
@@ -67,6 +71,12 @@ __all__ = [
     "RolloutTrajectory",
     "RolloutStep",
     "RolloutMetrics",
+    "TaskDescriptor",
+    "DatasetInfo",
+    "RubricInfo",
+    "RubricSection",
+    "InferenceInfo",
+    "LimitsInfo",
     "TaskInfo",
     "to_jsonable",
     "normalize_environment_api_key",
@@ -75,7 +85,6 @@ __all__ = [
     "normalize_vendor_keys",
     "get_openai_key_or_503",
     "get_groq_key_or_503",
-    "INTERACT_TOOL_SCHEMA",
     "prepare_for_openai",
     "prepare_for_groq",
     "inject_system_hint",