synth-ai 0.2.12__py3-none-any.whl → 0.2.13.dev1__py3-none-any.whl

synth_ai/judge_schemas.py

@@ -0,0 +1,127 @@
+"""
+Judge API Contract Schemas
+
+These schemas define the expected structure for requests and responses
+to the judge scoring endpoint at POST /api/judge/v1/score.
+
+This is the canonical contract that the backend MUST conform to.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+
+class CriterionScorePayload(BaseModel):
+    """Per-criterion score returned by the judge."""
+
+    score: float = Field(..., description="Numeric score for this criterion")
+    reason: str = Field(default="", description="Explanation for the score")
+    weight: float = Field(default=1.0, description="Weight of this criterion")
+    description: str = Field(default="", description="Description of the criterion")
+
+
+class ReviewPayload(BaseModel):
+    """Rubric review (event-level or outcome-level)."""
+
+    criteria: Dict[str, CriterionScorePayload] = Field(
+        default_factory=dict,
+        description="Map of criterion keys to their scores"
+    )
+    total: float = Field(default=0.0, description="Aggregated total score")
+    summary: Optional[str] = Field(None, description="Optional text summary")
+
+
+class JudgeScoreResponse(BaseModel):
+    """
+    Response body for POST /api/judge/v1/score.
+    
+    This is the canonical contract that judge backends MUST return.
+    """
+
+    status: Literal["ok", "failed"] = Field(default="ok", description="Request status")
+    event_reviews: List[ReviewPayload] = Field(
+        default_factory=list,
+        description="List of per-event rubric reviews (one per step)"
+    )
+    outcome_review: Optional[ReviewPayload] = Field(
+        None,
+        description="Optional outcome-level rubric review"
+    )
+    event_totals: List[float] = Field(
+        default_factory=list,
+        description="List of aggregated scores per event (matches event_reviews length)"
+    )
+    details: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional details (provider, latency, etc.)"
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Request metadata (provider, options, etc.)"
+    )
+
+    def aggregate_event_reward(self) -> float | None:
+        """
+        Aggregate all event totals into a single reward.
+        
+        Returns:
+            Sum of all event_totals, or None if empty
+        """
+        if not self.event_totals:
+            return None
+        return sum(self.event_totals)
+
+    def aggregate_outcome_reward(self) -> float | None:
+        """
+        Extract outcome reward from outcome_review.
+        
+        Returns:
+            outcome_review.total, or None if no outcome review
+        """
+        if self.outcome_review is None:
+            return None
+        return self.outcome_review.total
+
+
+# Request schemas for completeness
+
+class JudgeTaskApp(BaseModel):
+    """Task application metadata."""
+    
+    id: str = Field(..., description="Task app identifier")
+    base_url: Optional[str] = Field(None, description="Optional base URL for task app")
+
+
+class JudgeOptions(BaseModel):
+    """Judge provider and configuration options."""
+    
+    provider: Optional[str] = Field(None, description="Judge provider (e.g., 'openai', 'groq')")
+    model: Optional[str] = Field(None, description="Model identifier")
+    rubric_id: Optional[str] = Field(None, description="Rubric identifier")
+    event: bool = Field(True, description="Enable event-level judging")
+    outcome: bool = Field(True, description="Enable outcome-level judging")
+
+
+class JudgeTracePayload(BaseModel):
+    """Trace payload containing trajectory context."""
+    
+    event_history: List[Dict[str, Any]] = Field(..., description="List of events/steps")
+    markov_blanket_message_history: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description="Optional message history for context"
+    )
+    metadata: Dict[str, Any] = Field(default_factory=dict, description="Trace metadata")
+
+
+class JudgeScoreRequest(BaseModel):
+    """Request body for POST /api/judge/v1/score."""
+    
+    policy_name: str = Field(..., description="Name of the policy being evaluated")
+    task_app: JudgeTaskApp = Field(..., description="Task application metadata")
+    trace: JudgeTracePayload = Field(..., description="Trajectory trace to evaluate")
+    options: JudgeOptions = Field(default_factory=lambda: JudgeOptions(), description="Judge options")
+    rubric: Optional[Dict[str, Any]] = Field(None, description="Optional explicit rubric criteria")
+

synth_ai/rubrics/__init__.py

@@ -0,0 +1,22 @@
+"""
+Rubric utilities.
+
+Exposes helpers for validating rubric specifications that are used across
+Crafter-style judge configurations.
+"""
+
+from .validators import (
+    RubricCriterion,
+    RubricSpec,
+    ValidationError,
+    validate_rubric_dict,
+    validate_rubric_file,
+)
+
+__all__ = [
+    "RubricCriterion",
+    "RubricSpec",
+    "ValidationError",
+    "validate_rubric_dict",
+    "validate_rubric_file",
+]

synth_ai/rubrics/validators.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+from typing import Any, Iterable, Literal
+
+import pydantic
+
+
+class RubricCriterion(pydantic.BaseModel):
+    """Single scoring criterion within a rubric."""
+
+    id: str
+    description: str
+    weight: float
+    scale: str | None = None
+
+    @pydantic.field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("weight must be a finite number")
+        if value <= 0.0:
+            raise ValueError("weight must be positive")
+        if value > 1.0:
+            raise ValueError("weight must be <= 1.0")
+        return value
+
+    @pydantic.field_validator("id", "description", mode="before")
+    @classmethod
+    def _strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        return value
+
+
+class RubricSpec(pydantic.BaseModel):
+    """High-level rubric definition used by step-wise judges."""
+
+    version: str
+    goal_text: str
+    aggregation: Literal["weighted_sum"]
+    criteria: list[RubricCriterion]
+
+    @pydantic.model_validator(mode="after")
+    def _validate_weights(self) -> "RubricSpec":
+        if not self.criteria:
+            raise ValueError("rubric must declare at least one criterion")
+        total_weight = sum(criterion.weight for criterion in self.criteria)
+        if not math.isclose(total_weight, 1.0, abs_tol=1e-6, rel_tol=1e-6):
+            raise ValueError(
+                f"criterion weights must sum to 1 (got {total_weight:.6f})"
+            )
+        return self
+
+    @pydantic.field_validator("version")
+    @classmethod
+    def _non_empty_version(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("version string must not be empty")
+        return value
+
+    @pydantic.field_validator("goal_text")
+    @classmethod
+    def _non_empty_goal_text(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("goal_text must not be empty")
+        return value
+
+
+ValidationError = pydantic.ValidationError
+
+
+def validate_rubric_dict(payload: dict[str, Any]) -> RubricSpec:
+    """
+    Validate an in-memory rubric payload and return the parsed model.
+
+    Args:
+        payload: Dictionary representing the rubric JSON.
+    Returns:
+        Validated RubricSpec instance.
+    Raises:
+        ValidationError: If the payload is missing required fields or contains
+        invalid weights.
+    """
+
+    if not isinstance(payload, dict):
+        raise TypeError("rubric payload must be a dictionary")
+    return RubricSpec.model_validate(payload)
+
+
+def _load_payload_from_file(path: Path) -> dict[str, Any]:
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Unsupported rubric file type: {path}")
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text)
+
+
+def validate_rubric_file(path: Path) -> RubricSpec:
+    """
+    Load and validate a rubric file.
+
+    Args:
+        path: Path to a JSON rubric document.
+    Returns:
+        Validated RubricSpec instance.
+    """
+
+    payload = _load_payload_from_file(path)
+    return validate_rubric_dict(payload)
+
+
+def validate_rubric_files(paths: Iterable[Path]) -> list[RubricSpec]:
+    """
+    Validate multiple rubric files and return their parsed models.
+
+    Useful for bulk validation inside tests or CI checks.
+    """
+
+    validated: list[RubricSpec] = []
+    for path in paths:
+        validated.append(validate_rubric_file(path))
+    return validated

synth_ai/tracing_v3/serialization.py

@@ -0,0 +1,130 @@
+"""HTTP-safe serialization helpers for tracing v3.
+
+These utilities normalize tracing structures (including dataclasses) into
+JSON-serializable forms and provide a compact JSON encoder suitable for
+HTTP transmission to backend services.
+
+Design goals:
+- Preserve structure while ensuring standard-compliant JSON (no NaN/Infinity)
+- Handle common non-JSON types: datetime, Decimal, bytes, set/tuple, numpy scalars
+- Keep output compact (no unnecessary whitespace) while readable if needed
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from dataclasses import asdict, is_dataclass
+from datetime import date, datetime
+from decimal import Decimal
+from enum import Enum
+from typing import Any
+
+try:
+    import numpy as _np  # type: ignore
+except Exception:  # pragma: no cover - numpy optional at runtime
+    _np = None  # type: ignore
+
+
+def normalize_for_json(value: Any) -> Any:
+    """Return a JSON-serializable version of ``value``.
+
+    Rules:
+    - dataclass → dict (recursively normalized)
+    - datetime/date → ISO-8601 string (UTC-aware datetimes preserve tzinfo)
+    - Decimal → float (fallback to string if not finite)
+    - bytes/bytearray → base64 string (RFC 4648)
+    - set/tuple → list
+    - Enum → enum.value (normalized)
+    - numpy scalar → corresponding Python scalar
+    - float NaN/Inf/−Inf → None (to keep JSON standard compliant)
+    - dict / list → recursively normalized
+    - other primitives (str, int, bool, None, float) passed through
+    """
+
+    # Dataclasses
+    if is_dataclass(value) and not isinstance(value, type):
+        try:
+            return normalize_for_json(asdict(value))
+        except Exception:
+            # Fallback: best-effort conversion via __dict__
+            return normalize_for_json(getattr(value, "__dict__", {}))
+
+    # Mapping
+    if isinstance(value, dict):
+        return {str(k): normalize_for_json(v) for k, v in value.items()}
+
+    # Sequences
+    if isinstance(value, (list, tuple, set)):
+        return [normalize_for_json(v) for v in value]
+
+    # Datetime / Date
+    if isinstance(value, (datetime, date)):
+        return value.isoformat()
+
+    # Decimal
+    if isinstance(value, Decimal):
+        try:
+            f = float(value)
+            if f != f or f in (float("inf"), float("-inf")):
+                return str(value)
+            return f
+        except Exception:
+            return str(value)
+
+    # Bytes-like
+    if isinstance(value, (bytes, bytearray)):
+        return base64.b64encode(bytes(value)).decode("ascii")
+
+    # Enum
+    if isinstance(value, Enum):
+        return normalize_for_json(value.value)
+
+    # Numpy scalars / arrays
+    if _np is not None:
+        if isinstance(value, (_np.generic,)):  # type: ignore[attr-defined]
+            return normalize_for_json(value.item())
+        if isinstance(value, (_np.ndarray,)):
+            return normalize_for_json(value.tolist())
+
+    # Floats: sanitize NaN / Infinity to None
+    if isinstance(value, float):
+        if value != value or value in (float("inf"), float("-inf")):
+            return None
+        return value
+
+    return value
+
+
+def dumps_http_json(payload: Any) -> str:
+    """Dump ``payload`` into a compact, HTTP-safe JSON string.
+
+    - Recursively normalizes non-JSON types (see ``normalize_for_json``)
+    - Disallows NaN/Infinity per RFC 8259 (allow_nan=False)
+    - Uses compact separators and preserves Unicode (ensure_ascii=False)
+    """
+
+    normalized = normalize_for_json(payload)
+    return json.dumps(
+        normalized,
+        ensure_ascii=False,
+        allow_nan=False,
+        separators=(",", ":"),
+    )
+
+
+def serialize_trace_for_http(trace: Any) -> str:
+    """Serialize a tracing v3 session (or dict-like) to HTTP-safe JSON.
+
+    Accepts either a dataclass (e.g., SessionTrace) or a dict/list and
+    applies normalization and compact JSON encoding.
+    """
+
+    if is_dataclass(trace) and not isinstance(trace, type):
+        try:
+            return dumps_http_json(asdict(trace))
+        except Exception:
+            return dumps_http_json(getattr(trace, "__dict__", {}))
+    return dumps_http_json(trace)
+
+

{synth_ai-0.2.12.dist-info → synth_ai-0.2.13.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.12
+Version: 0.2.13.dev1
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT

{synth_ai-0.2.12.dist-info → synth_ai-0.2.13.dev1.dist-info}/RECORD

@@ -2,9 +2,24 @@ examples/__init__.py,sha256=S1h7WGBIgk2CmmSIqsE4nkhxv2XTniHSBUqKETgSNQI,543
 examples/analyze_semantic_words.sh,sha256=z-3qJBAlQuMrfRzoszofle_2b3Fy79rymvEAcoBQGXU,652
 examples/crafter_debug_render.py,sha256=McPkX8z6ffANOfoR0Xw26h81HNHtT0F5y4bzUhwYonQ,6261
 examples/run_crafter_demo.sh,sha256=7FNvooSgq-ezccGy5j_h_uRXObiQtcnybPcTwaVGDpo,392
+examples/agora_ex/README_MoE.md,sha256=EXM-tLE4WuqwmhdbgrRFf7lP0VhlszpDz4jO2j30klI,5770
+examples/agora_ex/__init__.py,sha256=oZU7CCBKQRTlIC1WeYA7QgArSQ5b6Kn4KDEu5KbMYyU,171
+examples/agora_ex/agora_ex.py,sha256=rjySIJVehB5lSYW_tV3fanq044uzzDo9o9EprQ5hpaM,2154
+examples/agora_ex/agora_ex_task_app.py,sha256=Va_A00AWT7byzKAujqEiF1vNRYjmXzkw_nMsCe8n2SU,20987
+examples/agora_ex/reward_fn_grpo-human.py,sha256=sgaUrWOt1malqX1GEN1fMpEvO1LPKWo30RO3fvO95OQ,4796
+examples/agora_ex/system_prompt_CURRENT.md,sha256=sz9txJsFzqG1mljbJH01iIYOynfGf98DNuSQHBu9RQw,2640
+examples/agora_ex/configs/rl_lora_qwen3_moe_2xh200.toml,sha256=VZBOahFd4aFj5SRl9_cfMqXn3sBRv8UQInQZ9LkNkD0,2922
+examples/agora_ex/task_app/agora_ex_task_app.py,sha256=Va_A00AWT7byzKAujqEiF1vNRYjmXzkw_nMsCe8n2SU,20987
+examples/agora_ex/task_app/reward_fn_grpo-human.py,sha256=sgaUrWOt1malqX1GEN1fMpEvO1LPKWo30RO3fvO95OQ,4796
+examples/agora_ex/task_app/system_prompt_CURRENT.md,sha256=sz9txJsFzqG1mljbJH01iIYOynfGf98DNuSQHBu9RQw,2640
 examples/dev/qwen3_32b_qlora_4xh100.toml,sha256=YXLo1I_eHamUHMjIvsD9Tx3_HXcLfjlasd6X2BF7i5g,766
-examples/multi_step/crafter_rl_lora.md,sha256=yv7C7lyBELT8OWufi8PtG-Z7Hqxr5KxVTmK8XUxOAIw,1033
-examples/multi_step/task_app_config_notes.md,sha256=nCyJYZwPI_5ifmuNw_cJATeRaO0aikk67tbG_J4WQns,21841
+examples/multi_step/crafter_rl_lora.md,sha256=sZHliCaYCq_T70JyTyK0jr-1yQ17zkZremj9GSeHG_I,2815
+examples/multi_step/sse_metrics_streaming_notes.md,sha256=2k5nNOB4IgXrfC0czdup5KryMOwbHT_HbSpS53UNX2U,15630
+examples/multi_step/task_app_config_notes.md,sha256=1ngz8aGCykA1iMCrVdc1yGbqXan530893Xzf4KK9xe4,22212
+examples/multi_step/configs/crafter_rl_outcome.toml,sha256=gZwpQBqRoGLkQnYvOk98dTjKO8EMUd3lg8daxXNoD-8,1378
+examples/multi_step/configs/crafter_rl_stepwise_hosted_judge.toml,sha256=4IorHrf9E8LcjDXVFWzh95AsFQabnruvhtSxplzUl1E,5047
+examples/multi_step/configs/crafter_rl_stepwise_shaped.toml,sha256=MjNyqzbWmer1S6UtiWHvzFLaJl5JX3yfwGcvnU_F4r8,1864
+examples/multi_step/configs/crafter_rl_stepwise_simple.toml,sha256=VUXFDFUMlni95Dtj1v9k52_LAl6Iq97ra4UTw6v2-60,1492
 examples/qwen_coder/README.md,sha256=xhoGKdIlRt8fO3F6HTpq6rzw3bs4ezDsGcRBNCz90Fs,3145
 examples/qwen_coder/_shared.py,sha256=oGUmx7gKkgkmToXHXAPocG72YKO9WGRjguVhmX3XE7s,3557
 examples/qwen_coder/generate_dataset.py,sha256=bfa1LMvV7jd2GmNkFE5rn2K1SETNEzMjv7SXD0C24K0,4484
@@ -85,7 +100,7 @@ examples/warming_up_to_rl/export_trace_sft.py,sha256=qcVOgZduYYImDv2xs-h_CoyMpX_
 examples/warming_up_to_rl/groq_test.py,sha256=vffVVTwQNypkqppxMLy17ySafoD2WKIK-Qf9w13lbyE,3138
 examples/warming_up_to_rl/manage_secrets.py,sha256=qLe_zglOLCd0dxPPCyQQ9CnhGkvtLOkBrZ0Xd4mmnIw,4372
 examples/warming_up_to_rl/readme.md,sha256=u2KX7grlDnOMRRrC3vzTej1K0OOmTdx5LHs2opTfLt4,6431
-examples/warming_up_to_rl/run_eval.py,sha256=-jd7_TtAyNKiHyH92slvDF-JzfCDH23M8q5CK-_usCo,26212
+examples/warming_up_to_rl/run_eval.py,sha256=zN4hZLJILulQNUw_WiDwEZUA3xIAcG6ewl7X_iNP-n8,31390
 examples/warming_up_to_rl/run_fft_and_save.py,sha256=RVbMuHUB0F7h_CO26hvAhEx5W3fqxVvokb4VlOiR6RQ,15095
 examples/warming_up_to_rl/run_local_rollout.py,sha256=uRfCR1E066fF6cQTajllb0Dzv0sU4iT6QOitZ_jRVHw,8783
 examples/warming_up_to_rl/run_local_rollout_modal.py,sha256=oGHqSCNUSAluJFlVkOKItONWh0wUu4UxRXmqnkD37E8,8948
@@ -98,16 +113,16 @@ examples/warming_up_to_rl/configs/crafter_fft_4b.toml,sha256=q_cnU3P-eGG_VFOepw9
 examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml,sha256=YP4HLWDh6iIvw6McPXw5kK1RUFQF4dvKP4yH5bHT5nI,678
 examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml,sha256=zQi31JYa83kW-ceEqDZi-7oajsCmEPrlJR57zN5ygO8,340
 examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml,sha256=6eeU1GVvK1cYSEuGXk-AhOwJLgRcf74CTOI5XlqNYBc,817
-examples/warming_up_to_rl/configs/eval_stepwise_complex.toml,sha256=sTfzF4z9IXFmReftdgHMw5k3aH78bdpSlF_ANyovsdo,941
+examples/warming_up_to_rl/configs/eval_stepwise_complex.toml,sha256=M1YGni8ng1RTJQR3xM65cv1jotNrJqseru47bd6omtY,1040
 examples/warming_up_to_rl/configs/eval_stepwise_consistent.toml,sha256=N5xt_9_AaVvJX6up90bSuXpF8Yt-cGJfmTA-Au3NY_4,591
 examples/warming_up_to_rl/configs/eval_stepwise_per_achievement.toml,sha256=ly42h8kIeSX8Y9mygC8mx7G_0KypUcPB9vqx8P-6QmQ,825
-examples/warming_up_to_rl/configs/eval_stepwise_simple.toml,sha256=ZNCryLgVCEkv8anP1pKkfNZ_X2kiloW4lmTqeJXtbxs,783
+examples/warming_up_to_rl/configs/eval_stepwise_simple.toml,sha256=g_Q8EsHOpatX0VfKKBHMGxX0T3yZv42_UzLhF5evGa8,882
 examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml,sha256=MrcjPlR-5s7XayBXl087QcGuR6484HcdSgS9yYUE5uo,1868
 examples/warming_up_to_rl/configs/rl_from_ft.toml,sha256=d1cIoLeC80NgOjn0Wohk0a5IXE_ImHVgMsxWPkyAFKQ,1381
 examples/warming_up_to_rl/old/event_rewards.md,sha256=gHJd3ZeYOnj4xPXt-7sSJamgOaJQ-BpfdaF-CKJK3-0,13450
 examples/warming_up_to_rl/old/notes.md,sha256=Y9Zs_tUb2Y6kv0MmGe-kAvGM9zCtEDY3Ccf5j7PoFGU,4468
 examples/warming_up_to_rl/task_app/README.md,sha256=xjt_l7BGJmAMwanKoLAhoy2cCVav_Hh5Z4iKGLUNz3E,1664
-examples/warming_up_to_rl/task_app/grpo_crafter.py,sha256=4GrgnVL0ifhJ12Q10tiyr3OTb2hafHBgCd5MUYNcfmY,25067
+examples/warming_up_to_rl/task_app/grpo_crafter.py,sha256=f9KxV2S90f90H6BH7MyuEayyq4-p2U00VRUv2WiqrXM,24328
 examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py,sha256=dw0OCq1guI8rH6yA5twSRUmFztbgl2SNwMW_7sOxYUA,5405
 examples/warming_up_to_rl/task_app/synth_envs_hosted/README.md,sha256=kJaN1do8V4XM2_g51WMI3edCDpv5zEw_nrMFtEwO1SQ,4614
 examples/warming_up_to_rl/task_app/synth_envs_hosted/__init__.py,sha256=KrGX5yedzYZQeKVt5FTSVzln52d1tsVGRtqFDId68zw,120
@@ -115,9 +130,9 @@ examples/warming_up_to_rl/task_app/synth_envs_hosted/branching.py,sha256=YpDgOaj
 examples/warming_up_to_rl/task_app/synth_envs_hosted/environment_routes.py,sha256=Y0iaLQ1nUkxBCD4aY39J4XEt66J-N4_J0uPj492i75g,49640
 examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py,sha256=hr1jblOUsx0bakgy6zXoEHROeUDoVPY1I66aNyyRQDg,7547
 examples/warming_up_to_rl/task_app/synth_envs_hosted/main.py,sha256=-GL__EH3Xr47vp3aD30XNXNrFPOj1bRLpOYQ-yvfwIU,2481
-examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py,sha256=6Wq62ni5qnW2fXuEuoGD0cMKAxjtNve_oG378Aqz1as,48568
+examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py,sha256=WOs0XzyEAAs_tbjSP-A4K5hw--IyMsnk5fSgEMh1GX0,48503
 examples/warming_up_to_rl/task_app/synth_envs_hosted/registry.py,sha256=5dN2Z-qVU4T_UUflHC9XGiIdDqFUl03G7uejcrYRbTE,5480
-examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py,sha256=8pzUbpdIYTmrfLPGxzwr0tgn7ng75ijEMY4_XONBMC4,84720
+examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py,sha256=Uuxy4nbVPRdmUm1RhQ_2xE6H0vg8u9LwQ60utbHeh4g,84409
 examples/warming_up_to_rl/task_app/synth_envs_hosted/test_agents.py,sha256=-TDfCs-LSlRgyWHGvO_6YAmjauwi1x4618D8CA9d8aA,5600
 examples/warming_up_to_rl/task_app/synth_envs_hosted/test_service.py,sha256=jFEtE3UqVCq552MSPliS2eO0pDbAS3tSDRJL0A-edTA,5111
 examples/warming_up_to_rl/task_app/synth_envs_hosted/utils.py,sha256=8Rnp0NTKTD_kPXg8zfpIy_we2hfk2agKPUVH_aqMsE0,1994
@@ -130,18 +145,19 @@ examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/react_agent.py
 examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/shared.py,sha256=NfhgJPkzjhA-bLDF6zYP57Cs1pnpqteJqzO3lsFOdMk,10074
 examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/tools.py,sha256=zmTxJ8IpTod7DRDCQhkLDw7KgFCeAVFbf4cQpuOgSgI,1788
 examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/__init__.py,sha256=TA47fqudhRMma0iANDvMotfC0U5YqJJQudeZFRGiPX4,179
-examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py,sha256=rXEnoZA04IWnEVNfORBaffELa4ub0dqes4sMAc3Nw_Y,27793
+examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py,sha256=R3lsxp2T55Y3YY4S4NqG6IjN5KCLyB7iNt2Php5Z9Xk,31754
 examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/__init__.py,sha256=1nwfPuiIFXPjj6JnxoudYq6GE4tpg5tiPL0uIpGsIUc,134
 examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/volume.py,sha256=1YJt5gpKhhJaFT8Cs7DdK_QDZCdUBAXQmpZfC5TX5Q4,6517
-synth_ai/__init__.py,sha256=pRBmoVw5oPO-vKrvHh9ioLk_RsbJTYxhpmkC95GMKfo,1342
+synth_ai/__init__.py,sha256=36R8UX48aD2ybWdrt79puVFDRyrDYGIC7gjwU0enrPI,2565
 synth_ai/__main__.py,sha256=Kh1xBKkTE5Vs2qNMtDuuOXerHUptMcOiF3YziOpC6DA,146
 synth_ai/demo_registry.py,sha256=vqt_E7-YdjXc-Q4qcY36wfvwHPGyYBb7lw_sPjKiH3w,9949
 synth_ai/handshake.py,sha256=Qxe0aLNcBVH5V02fNKq96GPF_2Z8GOAToDR_2HyaeaA,3739
 synth_ai/http.py,sha256=ACzDOkwa7kIniZVOZngPc6Zfp05h6rQE1oNqsWG_FKo,1038
 synth_ai/http_client.py,sha256=5AkwvGf7HpvYUpxr_IIM1xdsDjLYois7xTht5ueHYkk,5058
+synth_ai/judge_schemas.py,sha256=IGM-8ZHKwNKISgWyxk9JhQySEAu0rn_07HlF2NkTs4c,4536
 synth_ai/api/models/supported.py,sha256=vQu6Ahw1jn1X3-nL190pH_Z45tkQj8hdDyTTda8t5LQ,12671
 synth_ai/api/train/__init__.py,sha256=aBwlmrj1HWVwDcvUrX4TqpS0BgvkE7Pv7npoIZmtTFU,118
-synth_ai/api/train/builders.py,sha256=k8EQSX2jYFbiePn2p-2sdG5rAbfxeeIOxrYI4UE6288,11026
+synth_ai/api/train/builders.py,sha256=6vMMcFZSLlM5Xan2iUhsop7L51Su7j3PeK22mZcpQfE,12106
 synth_ai/api/train/cli.py,sha256=YZc_Scx-ClE6aJjVMEIzfsiBOlX7caSdg4mPGC-xPm0,23167
 synth_ai/api/train/config_finder.py,sha256=4touziSykzQ9YCYL2CjXUfojDcJdKO3vFnoWXQ1fOx8,7178
 synth_ai/api/train/env_resolver.py,sha256=X6ILO46UebkFJo0CDhzfVCb9y3HC-pOPIwkNhbGgvOo,11854
@@ -149,20 +165,24 @@ synth_ai/api/train/pollers.py,sha256=-rcVrGMN7Rj2HrzJ1IAgphwcrsCzpwwTy0KDk76Sx2A
 synth_ai/api/train/supported_algos.py,sha256=AqCtrfiJkocULlDtTPppKWuvOOzQ4tBbh-k3e9MEDZk,5156
 synth_ai/api/train/task_app.py,sha256=MiJ0DLFjqdFNmgk-YHaHVQt31Kxl91Exh9IFfk7GSxM,6439
 synth_ai/api/train/utils.py,sha256=_8C1uoXyXXuEmthquHH5XtowVQkf84Kde4jCdmfKnCk,6165
+synth_ai/api/train/configs/__init__.py,sha256=cmf4jS2pyTZQXw35b4OCox1MBhuUwDXTa51M01BA7Ck,915
+synth_ai/api/train/configs/rl.py,sha256=3WrgrNSQZ76zNPa9eyq0NQfgvlnHwF7S9qaz1ugUI34,3673
+synth_ai/api/train/configs/sft.py,sha256=aIxS_xF3Zaq5dc1BaEFWwbwuavxXH2gsJXQq1HlwmlM,2656
+synth_ai/api/train/configs/shared.py,sha256=EZ296HzlbOfp414ERbMdyO0s_9UvOKWs7PiQPlAEMjU,492
 synth_ai/cli/__init__.py,sha256=WhaUb9_UNcQBM1c7W3VzeUKUaBBSV31C0cwvHI-9UFM,2423
 synth_ai/cli/_modal_wrapper.py,sha256=DHVP-wQi_gdxWyeEiA_DO0nRiMv7XpaOJ3T3-R2pVoI,599
 synth_ai/cli/_storage.py,sha256=ABfpzyVnCNWFdAgzxvG0_6CGvEEl91BYNTcrTw_V3dY,680
 synth_ai/cli/_typer_patch.py,sha256=xBTshJadUQjtOr-PyGZUytpxyV28UvfU5DPWBCa_mTo,1562
 synth_ai/cli/balance.py,sha256=VTeJJ98JTA6Z7FujwNzRZAlTzO2BNcFueSnXJYV5f50,8432
 synth_ai/cli/calc.py,sha256=cAED6qRzc9g_eKjk7o_ZQsGKw6TA4Dib7NNze8e8L2s,2740
-synth_ai/cli/demo.py,sha256=FCsohnKHzNmqT1wZ4PzJ2kLXOF7Mr0jS_ZZrEM_nFpM,5610
+synth_ai/cli/demo.py,sha256=iGp9OASFBPj4lBgdWaA-KBTnkrJoDGsL6CpRCgXRAug,5383
 synth_ai/cli/legacy_root_backup.py,sha256=Ab3cVlOu3k3kLlrTc2SOtSTJTRxlwJ4JvZg9Ol2BbXg,15961
 synth_ai/cli/man.py,sha256=JQDon73ZkuKP9xr1_vRh5fjV9_b5xiUb7zNjny7ArB8,3765
 synth_ai/cli/recent.py,sha256=iY7vwVH8T6YwEOEC0QLenZSZXpyedrYc_epbP8WORzE,4415
-synth_ai/cli/rl_demo.py,sha256=SBH5t5EnRgpq0nZm9LXlAunC8CprXwZkkTzjhWDrXaQ,9257
+synth_ai/cli/rl_demo.py,sha256=bSwjLATZWlpUjG9AShwTjVYEbgLv8qwMWK46YQStt4k,8118
 synth_ai/cli/root.py,sha256=78oCkjtMaLSHEpHRbiaX-FEsdB3gieI7OrZcUMpqrSU,13740
 synth_ai/cli/status.py,sha256=vjr4ifL48pitKojBhuJdY4oIB8NnJ0RZF44fOzKS4SI,4635
-synth_ai/cli/task_apps.py,sha256=d49vSAOqk-aKhQH1dWXwOhcEu3my-tRsjbdOBRQ3Mp8,105691
+synth_ai/cli/task_apps.py,sha256=0Y8KiLmaacD41RMqejMekPFM2BiXfSfCtMD98rQ9R34,105830
 synth_ai/cli/traces.py,sha256=tyt2OUcREK82an9hc2Uq3H-xn0vWuTf_Pirt0CWkodg,6617
 synth_ai/cli/turso.py,sha256=y9aHehGA1VvaWkoZyBhu5fLulWUT65powrpvvVp7gpA,2301
 synth_ai/cli/watch.py,sha256=Wi5H2YdV3gsEVMd5YIte2LHMweWqLkGpsK3aopdjCQE,17481
@@ -171,7 +191,7 @@ synth_ai/config/base_url.py,sha256=c85LaABBrvsl8Fp8KH0LNtJJrpnUwlzA5Ywbuth8fHE,3
 synth_ai/core/experiment.py,sha256=5ErxMLOMDHgBjWJjId-uViCyxwfx3Ns5NVrgMkPEpZk,234
 synth_ai/core/system.py,sha256=s-Z7np2ISYmYc1r9YN-y2yb3cgRlOalrh0iaqnxeo84,206
 synth_ai/demos/core/__init__.py,sha256=A2FjhY7KXGtyzdQXqeTPCkEhHfrH-eQg6bvP8HaYhZM,36
-synth_ai/demos/core/cli.py,sha256=Nmkw-0v66xv1HON6IQHxvfYTDMx7b6ii2t9XO4EUMqE,66212
+synth_ai/demos/core/cli.py,sha256=QknHcnrMz2WYQLzVMvt6DOHRDyPjpcV3U6WMfOSRwgU,64723
 synth_ai/demos/demo_task_apps/__init__.py,sha256=fj5IgEaCWLjYx-8DpfzmUdHIEE2TlpauEDvJRgAnLy0,267
 synth_ai/demos/demo_task_apps/core.py,sha256=Pwh7Q9HV1njZWhb479nBXM9z57bXDveUV2ZapXaSuSU,15448
 synth_ai/demos/demo_task_apps/crafter/__init__.py,sha256=3SnNZTzBjGR9eudStcww259vPmzoFBHJL-M0GDUD7Qo,24
@@ -211,7 +231,7 @@ synth_ai/environments/examples/crafter_classic/debug_translation.py,sha256=47DEQ
 synth_ai/environments/examples/crafter_classic/engine.py,sha256=i13hDJe-6BXWFqXAsdjNCrfjU_HpwWLE5-4KNT-VSbs,25257
 synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py,sha256=IMmqt0S9wh0sdP42G_Q6W_2j22gKX4YQZVvuIqLzLWc,2802
 synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py,sha256=oC1y_CdnB_xit4FH72dMo8ljlZW_Mpa3cMpbDVCtqyU,11127
-synth_ai/environments/examples/crafter_classic/environment.py,sha256=hDRBVNDYVj1dJRFLYTYxHucDsjwF-ZXXUEQsv7UYSLg,21668
+synth_ai/environments/examples/crafter_classic/environment.py,sha256=Vw70KPCR_45S7Qewlk5vcSQrskOqFofn85PxoWskTzU,22287
 synth_ai/environments/examples/crafter_classic/taskset.py,sha256=9uZpQOllUqMupIBwlz2Jv38hBYqY09czi1LOonCTk5c,9843
 synth_ai/environments/examples/crafter_classic/trace_hooks_v3.py,sha256=lka-3tNuYsvYQ6acEvJjvBPkylbqgCzuCd2QMOjNCRs,7194
 synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py,sha256=wHT2pyOJ55Jlh9iq-UND4cd-GQdyKxQmo9ncCuQ_Jiw,10166
@@ -407,7 +427,10 @@ synth_ai/environments/tasks/filters.py,sha256=CJDXMUF4HDVQgdR3tIqHddUuvGmwwQG27O
 synth_ai/environments/tasks/utils.py,sha256=2UfWqN5WE7blfpdd8GcPuT56sDAPXzlF0pGUyGO_M1g,2576
 synth_ai/environments/v0_observability/history.py,sha256=gW0SozHmxeL6PFNaTAQ7em8BNUlObsRpP4_MJ3drMBE,54
 synth_ai/environments/v0_observability/log.py,sha256=B4LvbPh0a1_E5dki2PlKAVHESB8oCQz0CmDBvC0GwdQ,38
+synth_ai/evals/__init__.py,sha256=NBInP4HqnLUtI7USUwOqnar2c7JT4g8ngvm612CMuqA,306
 synth_ai/evals/base.py,sha256=Pu_h3oJAAEqJxgj5eigqbVhrNjAGJnBtlxpacSbGXCI,353
+synth_ai/evals/client.py,sha256=oTs0rNijPm5b5H4Ise7LsilXKiZJRMs4X_mCudGtGxA,2581
+synth_ai/evals/types.py,sha256=vpg2u4vLi7EqXJYgMM44TPXKKQKDxkcK3_KzYeHU6l0,930
 synth_ai/inference/__init__.py,sha256=5xwgFMKRfbLC7TX3lwMqGOZ8HHOEnXwlDqV_FLjsd5A,74
 synth_ai/inference/client.py,sha256=FA1h4rcMu7M0qCCCJ4yQTqC9dM1weVhDWylry8znnCc,1262
 synth_ai/jobs/client.py,sha256=_NtmBlm7LLZX22W6arfaa0DAq-0vsV-h_Ep2yHAiwnE,10125
@@ -435,6 +458,8 @@ synth_ai/learning/sft/client.py,sha256=9KxDcnwQl5FKZgXOUsEvsihFJUg2CHJA6sc791cyp
 synth_ai/learning/sft/config.py,sha256=pdUvwmhYMSIrFTD7-a9MglPwHevP4OSed9TuOzdjkOc,9791
 synth_ai/learning/sft/data.py,sha256=gM9ObMO7L3woKUVL2p-Pz6iTikS1Yr40vce5zRy2H84,9454
 synth_ai/lm/__init__.py,sha256=uXZFG6zFox_oV6FM7cJQxmnCNUBBa50ZuhdXCNxx0Lo,875
+synth_ai/rubrics/__init__.py,sha256=hymlNv1PezlMibEIzQWGBNy412lwE5dIPa98tiX6AiI,411
+synth_ai/rubrics/validators.py,sha256=DrvQKcHQwbt8thAEFeqvlFZMdaApkwZOHda6hdm6Va8,3613
 synth_ai/task/__init__.py,sha256=FyzWbFjQCXga_ZT1oWVLt2N58dHsLcM5sMu7kjoTpW0,2409
 synth_ai/task/auth.py,sha256=IUD3DcU-43qczWV9sSccXsjDOkZe8sJHmcQtues1YWQ,5652
 synth_ai/task/client.py,sha256=ccbcYRaZZ5MBMcfFdA8x2t7PjXEdvKuFmwRxrRl_8vk,5741
@@ -460,6 +485,7 @@ synth_ai/tracing_v3/llm_call_record_helpers.py,sha256=zZ0i6SB7l6WYud9mRLBTyZ4BJC
 synth_ai/tracing_v3/lm_call_record_abstractions.py,sha256=j2RGuXVaV_EXmIosuXRDjptJSlrDXwb8x06k2fF6lqo,9195
 synth_ai/tracing_v3/migration_helper.py,sha256=izm7SNHtG3VDv_5ZmMk_mmwKitmShxUK3joNFOArZIY,4177
 synth_ai/tracing_v3/replica_sync.py,sha256=H3ReiULH3McnGxdrH9BWkSeIOFfUQ6enHk5SWeuRV6k,8843
+synth_ai/tracing_v3/serialization.py,sha256=KiMkIeV5FyvOU6fgCV1F5hhOCY_0BsyQdNrfEDREmgY,4132
 synth_ai/tracing_v3/session_tracer.py,sha256=z9-gt7SK68X_q0aOHwWOtGOaGClpoWkUkAxAOwTKpI8,18996
 synth_ai/tracing_v3/utils.py,sha256=t-ZEyP03j8k7jNxtLsOSvzQnq8SXBaRPdK7jCgPdVdI,3450
 synth_ai/tracing_v3/examples/basic_usage.py,sha256=gSIWxeCYNe5WL38FjCaOCv0RvmxZ8SDiPWhA1-8XAeg,7342
@@ -576,9 +602,9 @@ synth_ai/v0/tracing_v3/abstractions.py,sha256=7IHKkzicFEJ4UXJbodK11Pi8YYq7Vt8Ev_
 synth_ai/v0/tracing_v3/decorators.py,sha256=fRyVBKEbxAPZoDsBH7t8OVLVixvgD_Lw_1ONF1TuZxU,108
 synth_ai/v0/tracing_v3/llm_call_record_helpers.py,sha256=JYSpGtW-RC-DnhUIPdVADNF3rK5x8Tl8DOymbmRI99I,121
 synth_ai/v0/tracing_v3/session_tracer.py,sha256=TSUzb_fUUlpwOqFE1X-Fs14nG2CNe7MZLKOvH7cV2kE,112
-synth_ai-0.2.12.dist-info/licenses/LICENSE,sha256=ynhjRQUfqA_RdGRATApfFA_fBAy9cno04sLtLUqxVFM,1069
-synth_ai-0.2.12.dist-info/METADATA,sha256=SOcZAekWv6vMdvP5lLzyVitE1zRdUZHGCN-YEZnsiP4,5424
-synth_ai-0.2.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-synth_ai-0.2.12.dist-info/entry_points.txt,sha256=Neq-3bT7TAijjgOIR77pKL-WYg6TWBDeO8pp_nL4vGY,91
-synth_ai-0.2.12.dist-info/top_level.txt,sha256=1moNHgctEUJ3F3eH3V-7FSMb2iTTze1V13dj1R04oUY,18
-synth_ai-0.2.12.dist-info/RECORD,,
+synth_ai-0.2.13.dev1.dist-info/licenses/LICENSE,sha256=ynhjRQUfqA_RdGRATApfFA_fBAy9cno04sLtLUqxVFM,1069
+synth_ai-0.2.13.dev1.dist-info/METADATA,sha256=7RpJWL82aHYN-jx-yaaJ9WVozXidWUR4tWS2MkNXxNk,5429
+synth_ai-0.2.13.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+synth_ai-0.2.13.dev1.dist-info/entry_points.txt,sha256=GSFXaJreq4PJXbixkUI0GHZwGh2dZDG5pYaoVmqr_KE,46
+synth_ai-0.2.13.dev1.dist-info/top_level.txt,sha256=1moNHgctEUJ3F3eH3V-7FSMb2iTTze1V13dj1R04oUY,18
+synth_ai-0.2.13.dev1.dist-info/RECORD,,

{synth_ai-0.2.12.dist-info → synth_ai-0.2.13.dev1.dist-info}/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 synth-ai = synth_ai.cli:cli
-synth-ai-demo = synth_ai.demos.core.cli:main