synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

synth_ai/environments/examples/sokoban/taskset.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 import logging
 import os
+from collections.abc import Iterable, Sequence
 from dataclasses import asdict, dataclass, fields
 from typing import List, Tuple
 from uuid import UUID, uuid4
@@ -18,6 +21,7 @@ from synth_ai.environments.tasks.core import (
     TaskInstanceMetadataFilter,
     TaskInstanceSet,
 )
+from synth_ai.task.contracts import TaskInfo
 
 logger = logging.getLogger(__name__)
 
@@ -96,6 +100,118 @@ class SokobanTaskInstance(TaskInstance):
         return cls(**filtered_data)
 
 
+def _base_task_info_template() -> TaskInfo:
+    return TaskInfo(
+        task={"id": "sokoban", "name": "Sokoban", "version": "1.0.0"},
+        environment="sokoban",
+        action_space={
+            "type": "tool_call",
+            "tools": [{"name": "interact", "schema": {"action": "int"}}],
+            "max_calls": 1,
+        },
+        observation={"summary": "Sokoban grid observation", "keys": ["grid", "player"]},
+        dataset={"id": "sokoban", "name": "Sokoban", "version": "1.0.0"},
+        rubric={"version": "1", "criteria_count": 1, "source": "inline"},
+        inference={"supports_proxy": False},
+        capabilities={"supports_rollout": True, "supports_env_lifecycle": True},
+        limits={"max_turns": 200},
+    )
+
+
+class SokobanTaskSet:
+    """Minimal helper compatible with Task App expectations."""
+
+    def __init__(self) -> None:
+        self._taskset: TaskInstanceSet | None = None
+        self._seed_index: dict[int, SokobanTaskInstance] = {}
+        self._base_info = _base_task_info_template()
+
+    async def _ensure_loaded(self) -> TaskInstanceSet:
+        if self._taskset is None:
+            dataset = await create_sokoban_taskset()
+            self._taskset = dataset
+            self._seed_index.clear()
+            for inst in dataset.instances:
+                try:
+                    seed_value = int(getattr(inst.metadata, "seed"))
+                except Exception:
+                    continue
+                # Keep the first instance encountered for a seed
+                self._seed_index.setdefault(seed_value, inst)
+        return self._taskset
+
+    def describe(self) -> dict[str, object]:
+        if not self._taskset:
+            return {"id": "sokoban", "name": "Sokoban"}
+        return {
+            "id": "sokoban",
+            "name": self._taskset.name,
+            "description": self._taskset.description,
+            "instance_count": len(self._taskset.instances),
+        }
+
+    async def provide_task_instances(self, seeds: Sequence[int]) -> Iterable[TaskInfo]:
+        await self._ensure_loaded()
+        if not seeds:
+            return []
+
+        infos: list[TaskInfo] = []
+        for raw_seed in seeds:
+            try:
+                seed_value = int(raw_seed)
+            except Exception:
+                continue
+
+            instance = self._seed_index.get(seed_value)
+            if instance is None:
+                # Attempt to construct on the fly; try configured difficulties in order
+                for difficulty in DIFFICULTY_CONFIGS:
+                    try:
+                        instance = await create_task_instance_from_seed(difficulty, seed_value)
+                        break
+                    except Exception:
+                        continue
+                if instance is None:
+                    continue
+                self._seed_index[seed_value] = instance
+
+            metadata = getattr(instance, "metadata", None)
+            base_info = self._base_info.model_copy(deep=True)
+
+            observation = dict(base_info.observation)
+            dataset_info = dict(base_info.dataset)
+            task_metadata = {"seed": seed_value}
+
+            if metadata is not None:
+                for key in ("difficulty", "num_boxes", "dim_room", "max_steps", "shortest_path_length"):
+                    value = getattr(metadata, key, None)
+                    if value is not None:
+                        observation[key] = value
+                        task_metadata[key] = value
+                dataset_info.update(
+                    {
+                        "seed": getattr(metadata, "seed", seed_value),
+                        "difficulty": getattr(metadata, "difficulty", None),
+                        "num_boxes": getattr(metadata, "num_boxes", None),
+                        "dim_room": getattr(metadata, "dim_room", None),
+                    }
+                )
+                generation_params = getattr(metadata, "generation_params", None)
+                if generation_params is not None:
+                    task_metadata["generation_params"] = generation_params
+
+            infos.append(
+                base_info.model_copy(
+                    update={
+                        "observation": observation,
+                        "dataset": dataset_info,
+                        "task_metadata": task_metadata,
+                    }
+                )
+            )
+        return infos
+
+
 async def create_sokoban_taskset() -> TaskInstanceSet:
     """Generates Sokoban task instances from pre-generated verified puzzles."""
     instances = []

synth_ai/environments/examples/verilog/engine.py

@@ -67,6 +67,16 @@ class VerilogStepPenaltyComponent(RewardComponent):
         return self.penalty
 
 
+class VerilogSubmitSuccessComponent(RewardComponent):
+    """Reward for successful submission (tests passed)."""
+    async def score(self, state: VerilogPublicState, action: Any) -> float:
+        if hasattr(action, "get") and action.get("type") == "submit":
+            # Check if submission passed
+            if action.get("passed", False):
+                return 10.0  # Large reward for completing the task correctly
+        return 0.0
+
+
 class VerilogEngine(StatefulEngine):
     """
     Stateful Verilog evaluation engine with persistent artifact snapshots.
@@ -81,6 +91,7 @@ class VerilogEngine(StatefulEngine):
             components=[
                 VerilogCompileSuccessComponent(),
                 VerilogSimulationPassComponent(),
+                VerilogSubmitSuccessComponent(),
                 VerilogStepPenaltyComponent(penalty=-0.01),
             ]
         )
@@ -284,13 +295,28 @@ class VerilogEngine(StatefulEngine):
 
     async def submit(self) -> Dict[str, Any]:
         """Submit solution for grading."""
-        # For now, simple check based on last simulation
-        # In a full implementation, this would call the task's verify method
+        # Check if the last simulation passed
+        # Parse the last simulation output to determine if tests passed
+        passed = False
+        detail = "No simulation run yet"
+        
+        if self._last_simulate_output:
+            stdout = self._last_simulate_output
+            passed = (
+                "ALL_TESTS_PASSED" in stdout
+                or ("Mismatches: 0 " in stdout and "samples" in stdout)
+                or ("no mismatches" in stdout.lower() and "errors" not in stdout.lower())
+            )
+            if passed:
+                detail = "All tests passed"
+            else:
+                detail = "Tests failed - please review simulation output"
+        
         return {
             "ok": True,
             "type": "submit",
-            "passed": True,  # Placeholder
-            "detail": "Submission processed",
+            "passed": passed,
+            "detail": detail,
             "submitted": True,
         }
 

synth_ai/evals/client.py

@@ -1,11 +1,11 @@
-from __future__ import annotations
-
 """Experimental Judge API client.
 
 This surface is experimental and subject to change without notice.
 Set environment variable `SYNTH_SILENCE_EXPERIMENTAL=1` to silence warnings.
 """
 
+from __future__ import annotations
+
 import os
 import warnings
 from typing import Any, Literal, TypedDict
@@ -13,73 +13,70 @@ from typing import Any, Literal, TypedDict
 from synth_ai.http import AsyncHttpClient, HTTPError
 from synth_ai.tracing_v3.serialization import normalize_for_json
 
-
 Provider = Literal["groq", "gemini"]
 
 
 class JudgeOptions(TypedDict, total=False):
-	event: bool
-	outcome: bool
-	rubric_id: str
-	rubric_overrides: dict[str, Any]
-	provider: Provider
-	model: str
-	max_concurrency: int
+    event: bool
+    outcome: bool
+    rubric_id: str
+    rubric_overrides: dict[str, Any]
+    provider: Provider
+    model: str
+    max_concurrency: int
 
 
 class JudgeScoreResponse(TypedDict, total=False):
-	status: str
-	event_rewards: list[dict[str, Any]]
-	outcome_reward: dict[str, Any]
-	details: dict[str, Any]
+    status: str
+    event_rewards: list[dict[str, Any]]
+    outcome_reward: dict[str, Any]
+    details: dict[str, Any]
 
 
 class JudgeClient:
-	def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None:
-		_silence = (os.getenv("SYNTH_SILENCE_EXPERIMENTAL") or "").strip().lower()
-		if _silence not in {"1", "true", "t", "yes", "y", "on"}:
-			warnings.warn(
-				"Experimental API: synth_ai.evals.JudgeClient is experimental and may change without notice.",
-				UserWarning,
-				stacklevel=2,
-			)
-		self._base = base_url.rstrip("/")
-		self._key = api_key
-		self._timeout = timeout
-
-	async def score(
-		self,
-		*,
-		trace: dict[str, Any] | Any,
-		policy_name: str,
-		task_app_id: str,
-		options: JudgeOptions,
-		task_app_base_url: str | None = None,
-	) -> JudgeScoreResponse:
-		body = {
-			"policy_name": policy_name,
-			"task_app": {"id": task_app_id, **({"base_url": task_app_base_url} if task_app_base_url else {})},
-			"trace": normalize_for_json(trace),
-			"options": options or {},
-		}
-		try:
-			async with AsyncHttpClient(self._base, self._key, timeout=self._timeout) as http:
-				js = await http.post_json("/api/judge/v1/score", json=body)
-				if not isinstance(js, dict):
-					raise ValueError("invalid_judge_response_shape")
-				return js  # type: ignore[return-value]
-		except HTTPError as e:  # map to friendlier exceptions
-			status = int(getattr(e, "status", 0) or 0)
-			if status in (400, 422):
-				raise ValueError(f"judge_validation_error: {e.detail}") from e
-			if status in (401, 403):
-				raise PermissionError(f"judge_auth_error: {e.detail}") from e
-			if status == 404:
-				raise FileNotFoundError(f"judge_route_not_found: {e.detail}") from e
-			if status == 429:
-				raise Exception("judge_rate_limited") from e  # replace with RetryLater in future
-			if status >= 500:
-				raise Exception("judge_transient_error") from e  # replace with TransientError in future
-			raise
-
+    def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None:
+        _silence = (os.getenv("SYNTH_SILENCE_EXPERIMENTAL") or "").strip().lower()
+        if _silence not in {"1", "true", "t", "yes", "y", "on"}:
+            warnings.warn(
+                "Experimental API: synth_ai.evals.JudgeClient is experimental and may change without notice.",
+                UserWarning,
+                stacklevel=2,
+            )
+        self._base = base_url.rstrip("/")
+        self._key = api_key
+        self._timeout = timeout
 
+    async def score(
+        self,
+        *,
+        trace: dict[str, Any] | Any,
+        policy_name: str,
+        task_app_id: str,
+        options: JudgeOptions,
+        task_app_base_url: str | None = None,
+    ) -> JudgeScoreResponse:
+        body = {
+            "policy_name": policy_name,
+            "task_app": {"id": task_app_id, **({"base_url": task_app_base_url} if task_app_base_url else {})},
+            "trace": normalize_for_json(trace),
+            "options": options or {},
+        }
+        try:
+            async with AsyncHttpClient(self._base, self._key, timeout=self._timeout) as http:
+                js = await http.post_json("/api/judge/v1/score", json=body)
+                if not isinstance(js, dict):
+                    raise ValueError("invalid_judge_response_shape")
+                return js  # type: ignore[return-value]
+        except HTTPError as err:  # map to friendlier exceptions
+            status = int(getattr(err, "status", 0) or 0)
+            if status in (400, 422):
+                raise ValueError(f"judge_validation_error: {err.detail}") from err
+            if status in (401, 403):
+                raise PermissionError(f"judge_auth_error: {err.detail}") from err
+            if status == 404:
+                raise FileNotFoundError(f"judge_route_not_found: {err.detail}") from err
+            if status == 429:
+                raise Exception("judge_rate_limited") from err  # replace with RetryLater in future
+            if status >= 500:
+                raise Exception("judge_transient_error") from err  # replace with TransientError in future
+            raise

synth_ai/jobs/client.py

@@ -1,20 +1,32 @@
 from __future__ import annotations
 
 import importlib
-from typing import Any
+from collections.abc import Callable
+from typing import Any, cast
 
 try:
-    normalize_model_identifier = importlib.import_module("synth_ai.api.models.supported").normalize_model_identifier
+    _supported_module = cast(
+        Any, importlib.import_module("synth_ai.api.models.supported")
+    )
+    normalize_model_identifier = cast(
+        Callable[[str], str], _supported_module.normalize_model_identifier
+    )
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load supported model utilities") from exc
 
 try:
-    AsyncHttpClient = importlib.import_module("synth_ai.http").AsyncHttpClient
+    _http_module = cast(Any, importlib.import_module("synth_ai.http"))
+    AsyncHttpClient = _http_module.AsyncHttpClient
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load HTTP client") from exc
 
 try:
-    prepare_sft_job_payload = importlib.import_module("synth_ai.learning.sft.config").prepare_sft_job_payload
+    _sft_config_module = cast(
+        Any, importlib.import_module("synth_ai.learning.sft.config")
+    )
+    prepare_sft_job_payload = cast(
+        Callable[..., dict[str, Any]], _sft_config_module.prepare_sft_job_payload
+    )
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load SFT configuration helpers") from exc
 

synth_ai/judge_schemas.py

@@ -9,7 +9,7 @@ This is the canonical contract that the backend MUST conform to.
 
 from __future__ import annotations
 
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Literal
 
 from pydantic import BaseModel, Field
 
@@ -26,12 +26,12 @@ class CriterionScorePayload(BaseModel):
 class ReviewPayload(BaseModel):
     """Rubric review (event-level or outcome-level)."""
 
-    criteria: Dict[str, CriterionScorePayload] = Field(
+    criteria: dict[str, CriterionScorePayload] = Field(
         default_factory=dict,
         description="Map of criterion keys to their scores"
     )
     total: float = Field(default=0.0, description="Aggregated total score")
-    summary: Optional[str] = Field(None, description="Optional text summary")
+    summary: str | None = Field(None, description="Optional text summary")
 
 
 class JudgeScoreResponse(BaseModel):
@@ -42,23 +42,23 @@ class JudgeScoreResponse(BaseModel):
     """
 
     status: Literal["ok", "failed"] = Field(default="ok", description="Request status")
-    event_reviews: List[ReviewPayload] = Field(
+    event_reviews: list[ReviewPayload] = Field(
         default_factory=list,
         description="List of per-event rubric reviews (one per step)"
     )
-    outcome_review: Optional[ReviewPayload] = Field(
+    outcome_review: ReviewPayload | None = Field(
         None,
         description="Optional outcome-level rubric review"
     )
-    event_totals: List[float] = Field(
+    event_totals: list[float] = Field(
         default_factory=list,
         description="List of aggregated scores per event (matches event_reviews length)"
     )
-    details: Dict[str, Any] = Field(
+    details: dict[str, Any] = Field(
         default_factory=dict,
         description="Additional details (provider, latency, etc.)"
     )
-    metadata: Dict[str, Any] = Field(
+    metadata: dict[str, Any] = Field(
         default_factory=dict,
         description="Request metadata (provider, options, etc.)"
     )
@@ -92,15 +92,15 @@ class JudgeTaskApp(BaseModel):
     """Task application metadata."""
     
     id: str = Field(..., description="Task app identifier")
-    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
+    base_url: str | None = Field(None, description="Optional base URL for task app")
 
 
 class JudgeOptions(BaseModel):
     """Judge provider and configuration options."""
     
-    provider: Optional[str] = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
-    model: Optional[str] = Field(None, description="Model identifier")
-    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
+    provider: str | None = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: str | None = Field(None, description="Model identifier")
+    rubric_id: str | None = Field(None, description="Rubric identifier")
     event: bool = Field(True, description="Enable event-level judging")
     outcome: bool = Field(True, description="Enable outcome-level judging")
 
@@ -108,12 +108,12 @@ class JudgeOptions(BaseModel):
 class JudgeTracePayload(BaseModel):
     """Trace payload containing trajectory context."""
     
-    event_history: List[Dict[str, Any]] = Field(..., description="List of events/steps")
-    markov_blanket_message_history: List[Dict[str, Any]] = Field(
+    event_history: list[dict[str, Any]] = Field(..., description="List of events/steps")
+    markov_blanket_message_history: list[dict[str, Any]] = Field(
         default_factory=list,
         description="Optional message history for context"
     )
-    metadata: Dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
 
 
 class JudgeScoreRequest(BaseModel):
@@ -123,5 +123,5 @@ class JudgeScoreRequest(BaseModel):
     task_app: JudgeTaskApp = Field(..., description="Task application metadata")
     trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
     options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
-    rubric: Optional[Dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
+    rubric: dict[str, Any] | None = Field(None, description="Optional explicit rubric criteria")
 

synth_ai/task/__init__.py

@@ -5,6 +5,9 @@ from .auth import (
 )
 from .client import TaskAppClient
 from .contracts import (
+    DatasetInfo,
+    InferenceInfo,
+    LimitsInfo,
     RolloutEnvSpec,
     RolloutMetrics,
     RolloutPolicySpec,
@@ -14,8 +17,10 @@ from .contracts import (
     RolloutSafetyConfig,
     RolloutStep,
     RolloutTrajectory,
-    TaskAppContract,
+    RubricInfo,
+    RubricSection,
     TaskAppEndpoints,
+    TaskDescriptor,
     TaskInfo,
 )
 from .datasets import TaskDatasetRegistry, TaskDatasetSpec
@@ -23,7 +28,6 @@ from .errors import error_payload, http_exception, json_error_response
 from .health import task_app_health
 from .json import to_jsonable
 from .proxy import (
-    INTERACT_TOOL_SCHEMA,
     extract_message_text,
     inject_system_hint,
     parse_tool_call_from_text,
@@ -46,7 +50,7 @@ from .server import (
     create_task_app,
     run_task_app,
 )
-from .validators import validate_task_app_url
+from .validators import validate_task_app_endpoint, validate_task_app_url
 from .vendors import (
     get_groq_key_or_503,
     get_openai_key_or_503,
@@ -55,8 +59,8 @@ from .vendors import (
 
 __all__ = [
     "validate_task_app_url",
+    "validate_task_app_endpoint",
     "task_app_health",
-    "TaskAppContract",
     "TaskAppEndpoints",
     "RolloutEnvSpec",
     "RolloutPolicySpec",
@@ -67,6 +71,12 @@ __all__ = [
     "RolloutTrajectory",
     "RolloutStep",
     "RolloutMetrics",
+    "TaskDescriptor",
+    "DatasetInfo",
+    "RubricInfo",
+    "RubricSection",
+    "InferenceInfo",
+    "LimitsInfo",
     "TaskInfo",
     "to_jsonable",
     "normalize_environment_api_key",
@@ -75,7 +85,6 @@ __all__ = [
     "normalize_vendor_keys",
     "get_openai_key_or_503",
     "get_groq_key_or_503",
-    "INTERACT_TOOL_SCHEMA",
     "prepare_for_openai",
     "prepare_for_groq",
     "inject_system_hint",