synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/task/contracts.py

@@ -1,236 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class RolloutMode(str, Enum):
-    """Mode controls how rollout infrastructure processes inference URLs."""
-    RL = "rl"
-    EVAL = "eval"
-
-
-@dataclass(frozen=True)
-class TaskAppEndpoints:
-    """Required Task App endpoints used by RL trainers and clients.
-
-    Task Apps run as lightweight HTTP services (often on Modal) that expose these
-    standard endpoints. Additional endpoints (proxies, debug routes) may be added
-    by individual task apps as needed.
-    """
-
-    root: str = "/"
-    health: str = "/health"
-    info: str = "/info"
-    task_info: str = "/task_info"
-    rollout: str = "/rollout"
-
-
-# --- Unified rollout schema used by Task App services and SDK utilities ---
-
-
-class RolloutEnvSpec(BaseModel):
-    env_id: str | None = None
-    env_name: str | None = None
-    config: dict[str, Any] = Field(default_factory=dict)
-    seed: int | None = None
-
-
-class RolloutPolicySpec(BaseModel):
-    policy_id: str | None = None
-    policy_name: str | None = None
-    config: dict[str, Any] = Field(default_factory=dict)
-
-
-class RolloutRecordConfig(BaseModel):
-    trajectories: bool = True
-    logprobs: bool = False
-    value: bool = False
-    return_trace: bool = False
-    trace_format: Literal["compact", "full", "structured"] = "compact"
-
-
-class RolloutSafetyConfig(BaseModel):
-    max_ops: int = 100000
-    max_time_s: float = 3600.0
-
-
-class RolloutRequest(BaseModel):
-    run_id: str
-    env: RolloutEnvSpec
-    policy: RolloutPolicySpec
-    ops: list[dict[str, Any]] | list[str]
-    record: RolloutRecordConfig = RolloutRecordConfig()
-    on_done: str = "reset"
-    safety: RolloutSafetyConfig = RolloutSafetyConfig()
-    training_session_id: str | None = None
-    synth_base_url: str | None = None
-    mode: RolloutMode  # Required: explicit RL vs EVAL mode
-
-
-class RolloutStep(BaseModel):
-    """Single step in a rollout trajectory.
-    
-    DEPRECATED: This is part of the legacy trajectory format. New code should
-    consume v3 traces (RolloutResponse.trace) instead. See monorepo/trace_single_source.txt
-    for migration plan.
-    """
-    obs: dict[str, Any]
-    tool_calls: list[dict[str, Any]]
-    reward: float | None = None
-    done: bool = False
-    truncated: bool | None = None
-    info: dict[str, Any] | None = None
-
-
-class RolloutTrajectory(BaseModel):
-    """Legacy trajectory format for rollout results.
-    
-    DEPRECATED: This format duplicates data already present in v3 traces and will
-    be removed once training code migrates to consuming RolloutResponse.trace.
-    
-    Current state:
-    - Task apps emit BOTH this format AND v3 traces (dual serialization)
-    - Training code (GSPO) reads from this format
-    - Eval/filter tools read from v3 traces
-    
-    Migration plan:
-    - Phase 1: Training code learns to read from v3 traces (with fallback to this)
-    - Phase 2: Make this field optional once training is migrated
-    - Phase 3: Remove this field entirely and delete this class
-    
-    See: monorepo/trace_single_source.txt for full migration plan and timeline.
-    
-    Why v3 traces are better:
-    - Single source of truth (no duplication/drift)
-    - Richer data: token IDs, logprobs, reasoning, timing, images
-    - Built-in audit trail and replay capability
-    - Standard schema across all Synth AI tooling
-    """
-    env_id: str
-    policy_id: str
-    steps: list[RolloutStep]
-    final: dict[str, Any] | None = None
-    length: int
-    
-    # Required for trace correlation with inference mesh (optional initially for backward compat)
-    # See: monorepo/INFERENCE_URL_REQUIREMENT_PLAN.md and trace_creation_and_judgement.txt
-    inference_url: str
-    
-    decision_samples: list[dict[str, Any]] | None = None
-
-
-class RolloutMetrics(BaseModel):
-    episode_returns: list[float]
-    mean_return: float
-    num_steps: int
-    num_episodes: int = 0
-    outcome_score: float | None = None
-    events_score: float | None = None
-    details: dict[str, Any] = Field(default_factory=dict)
-
-
-class RolloutResponse(BaseModel):
-    """Response from a rollout execution.
-    
-    Contains both legacy trajectory format (for backward compatibility) and
-    modern v3 trace format (preferred going forward).
-    """
-    run_id: str
-    
-    # DEPRECATED: Legacy format maintained for training code compatibility.
-    # Will be removed once training migrates to reading from `trace` field.
-    # See: monorepo/trace_single_source.txt for migration plan.
-    trajectories: list[RolloutTrajectory]
-    
-    branches: dict[str, list[str]] = Field(default_factory=dict)
-    metrics: RolloutMetrics
-    aborted: bool = False
-    ops_executed: int = 0
-    
-    # OPTIONAL: correlation ID for linking rollout to inference traces
-    # If not provided, trainer will infer it from trajectory.inference_url ?cid=... parameter
-    trace_correlation_id: str | None = None
-    
-    # PREFERRED: v3 trace format (SessionTrace). This is the single source of truth
-    # for rollout data and should be used by all new code. Contains richer data than
-    # trajectories including token IDs, logprobs, timing, and multimodal content.
-    trace: dict[str, Any] | None = None
-    pipeline_metadata: dict[str, Any] = Field(default_factory=dict)
-
-
-class _ExtraAllowModel(BaseModel):
-    """Base helper that preserves unknown keys while still exposing typed attributes."""
-
-    model_config = ConfigDict(extra="allow")
-
-
-class TaskDescriptor(_ExtraAllowModel):
-    """Human-readable task identifiers shown in UIs and logs."""
-
-    id: str
-    name: str
-    description: str | None = None
-    version: str | None = None
-
-
-class DatasetInfo(_ExtraAllowModel):
-    """Metadata about the prompt/task dataset powering the environment."""
-
-    id: str | None = None
-    name: str | None = None
-    version: str | None = None
-    splits: list[str] | None = None
-    default_split: str | None = None
-    description: str | None = None
-
-
-class RubricCriterion(_ExtraAllowModel):
-    id: str
-    description: str
-    weight: float | None = None
-
-
-class RubricSection(_ExtraAllowModel):
-    name: str
-    criteria: list[RubricCriterion] = Field(default_factory=list)
-
-
-class RubricInfo(_ExtraAllowModel):
-    """Outcome and event scoring definitions used by judges."""
-
-    outcome: RubricSection | None = None
-    events: RubricSection | None = None
-
-
-class InferenceInfo(_ExtraAllowModel):
-    """Recommended defaults for policy model routing."""
-
-    model: str | None = None
-    inference_url: str | None = None
-
-
-class LimitsInfo(_ExtraAllowModel):
-    """Operational limits the environment enforces."""
-
-    max_turns: int | None = None
-    max_response_tokens: int | None = None
-    timeout_seconds: int | None = None
-
-
-class TaskInfo(_ExtraAllowModel):
-    """Static metadata describing the capabilities of a Task App task."""
-
-    task: TaskDescriptor
-    environment: str
-    dataset: DatasetInfo
-    rubric: RubricInfo
-    inference: InferenceInfo
-    limits: LimitsInfo
-    task_metadata: dict[str, Any] = Field(
-        default_factory=dict,
-        description="Task-specific extras (e.g. prompt version info, documentation links).",
-    )

synth_ai/task/datasets.py

@@ -1,108 +0,0 @@
-"""Dataset registry and helpers shared by Task Apps."""
-
-from __future__ import annotations
-
-from collections.abc import Callable, Hashable
-from typing import Any
-
-from pydantic import BaseModel, Field, field_validator
-
-
-class TaskDatasetSpec(BaseModel):
-    """Declarative metadata describing a dataset that a Task App exposes."""
-
-    id: str
-    name: str
-    version: str | None = None
-    splits: list[str] = Field(default_factory=list)
-    default_split: str | None = None
-    cardinality: int | None = None
-    description: str | None = None
-
-    @field_validator("default_split")
-    @classmethod
-    def _validate_default_split(cls, value: str | None, info):
-        values = info.data if hasattr(info, "data") else {}  # type: ignore[attr-defined]
-        if value and value not in (values.get("splits") or []):
-            raise ValueError("default_split must be one of splits when provided")
-        return value
-
-
-RegistryLoader = Callable[[TaskDatasetSpec], Any]
-
-
-class TaskDatasetRegistry:
-    """Lightweight registry mapping dataset specs to loader callables."""
-
-    def __init__(self) -> None:
-        self._entries: dict[str, tuple[TaskDatasetSpec, RegistryLoader, bool]] = {}
-        self._cache: dict[Hashable, Any] = {}
-
-    def register(
-        self, spec: TaskDatasetSpec, loader: RegistryLoader, *, cache: bool = True
-    ) -> None:
-        """Register a dataset loader and its metadata."""
-
-        self._entries[spec.id] = (spec, loader, cache)
-
-    def describe(self, dataset_id: str) -> TaskDatasetSpec:
-        if dataset_id not in self._entries:
-            raise KeyError(f"Dataset not registered: {dataset_id}")
-        spec, _, _ = self._entries[dataset_id]
-        return spec
-
-    def list(self) -> list[TaskDatasetSpec]:
-        return [entry[0] for entry in self._entries.values()]
-
-    def get(self, spec: TaskDatasetSpec | str) -> Any:
-        """Return dataset materialisation (with optional caching)."""
-
-        if isinstance(spec, str):
-            if spec not in self._entries:
-                raise KeyError(f"Dataset not registered: {spec}")
-            base_spec, loader, cache_enabled = self._entries[spec]
-            effective_spec = base_spec
-        else:
-            if spec.id not in self._entries:
-                raise KeyError(f"Dataset not registered: {spec.id}")
-            base_spec, loader, cache_enabled = self._entries[spec.id]
-            effective_spec = base_spec.model_copy(update=spec.model_dump(exclude_unset=True))
-
-        cache_key: Hashable = (
-            effective_spec.id,
-            effective_spec.version,
-            effective_spec.default_split,
-        )
-        if cache_enabled:
-            if cache_key not in self._cache:
-                self._cache[cache_key] = loader(effective_spec)
-            return self._cache[cache_key]
-        return loader(effective_spec)
-
-    @staticmethod
-    def ensure_split(spec: TaskDatasetSpec, split: str | None) -> str:
-        """Validate that `split` exists on the spec; return a concrete split."""
-
-        if not spec.splits:
-            return split or spec.default_split or "default"
-        if split is None:
-            if spec.default_split:
-                return spec.default_split
-            raise ValueError(f"split must be provided for dataset {spec.id}")
-        if split not in spec.splits:
-            raise ValueError(f"Unknown split '{split}' for dataset {spec.id}")
-        return split
-
-    @staticmethod
-    def normalise_seed(seed: Any, *, cardinality: int | None = None) -> int:
-        """Normalise arbitrary seed input into a bounded non-negative integer."""
-
-        try:
-            value = int(seed)
-        except Exception as exc:  # pragma: no cover - defensive
-            raise ValueError(f"Seed must be convertible to int (got {seed!r})") from exc
-        if value < 0:
-            value = abs(value)
-        if cardinality and cardinality > 0:
-            value = value % cardinality
-        return value

synth_ai/task/proxy.py

@@ -1,251 +0,0 @@
-"""Shared helpers for Task App proxy endpoints (OpenAI, Groq, etc.).
-
-The proxy is tool-agnostic - each task app provides its own tools schema.
-"""
-
-from __future__ import annotations
-
-import copy
-import json
-import re
-from typing import Any
-
-_REMOVE_FIELDS = {
-    "stop_after_tool_calls",
-    "thinking_mode",
-    "thinking_budget",
-    "reasoning",
-}
-_REMOVE_SAMPLING_FIELDS = {"temperature", "top_p"}
-_GPT5_MIN_COMPLETION_TOKENS = 16000
-
-
-def prepare_for_openai(model: str | None, payload: dict[str, Any]) -> dict[str, Any]:
-    """Sanitise an OpenAI chat completions payload for Task App usage.
-    
-    The task app is responsible for providing tools in the payload.
-    This function only handles model-specific parameter normalization.
-    """
-
-    sanitized = copy.deepcopy(payload)
-    for field in _REMOVE_FIELDS:
-        sanitized.pop(field, None)
-
-    if model and "gpt-5" in model:
-        max_tokens = sanitized.pop("max_tokens", None)
-        if "max_completion_tokens" not in sanitized and isinstance(max_tokens, int):
-            sanitized["max_completion_tokens"] = max_tokens
-        elif max_tokens is not None:
-            sanitized.setdefault("max_completion_tokens", max_tokens)
-        for field in _REMOVE_SAMPLING_FIELDS:
-            sanitized.pop(field, None)
-        mct = sanitized.get("max_completion_tokens")
-        if not isinstance(mct, int) or mct < _GPT5_MIN_COMPLETION_TOKENS:
-            sanitized["max_completion_tokens"] = _GPT5_MIN_COMPLETION_TOKENS
-        
-        # Set tool_choice to first provided tool (task app must provide tools)
-        # If tool_choice not already set and tools are provided, use the first one
-        if "tool_choice" not in sanitized:
-            tools = sanitized.get("tools", [])
-            if isinstance(tools, list) and tools:
-                first_func = tools[0].get("function", {})
-                if isinstance(first_func, dict) and "name" in first_func:
-                    sanitized["tool_choice"] = {"type": "function", "function": {"name": first_func["name"]}}
-        
-        sanitized["parallel_tool_calls"] = False
-
-    return sanitized
-
-
-def prepare_for_groq(model: str | None, payload: dict[str, Any]) -> dict[str, Any]:
-    """Groq uses the OpenAI schema; reuse most normalisation rules."""
-
-    sanitized = prepare_for_openai(model, payload)
-    # Groq supports `max_tokens`; prefer their native parameter when present
-    if (
-        model
-        and "gpt-5" not in model
-        and "max_completion_tokens" in sanitized
-        and "max_tokens" not in payload
-    ):
-        sanitized["max_tokens"] = sanitized.pop("max_completion_tokens")
-    return sanitized
-
-
-def inject_system_hint(payload: dict[str, Any], hint: str) -> dict[str, Any]:
-    """Insert or augment a system message with the provided hint (idempotent)."""
-
-    if not hint:
-        return payload
-    cloned = copy.deepcopy(payload)
-    messages = cloned.get("messages")
-    if not isinstance(messages, list):
-        return cloned
-    if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system":
-        content = messages[0].get("content")
-        if isinstance(content, str) and hint not in content:
-            messages[0] = dict(messages[0])
-            messages[0]["content"] = content.rstrip() + ("\n\n" if content else "") + hint
-    else:
-        messages.insert(0, {"role": "system", "content": hint})
-    cloned["messages"] = messages
-    return cloned
-
-
-def extract_message_text(message: Any) -> str:
-    """Return best-effort text from an OpenAI-style message structure."""
-
-    if message is None:
-        return ""
-    if isinstance(message, str):
-        return message
-    if isinstance(message, list):
-        parts = [extract_message_text(part) for part in message]
-        return "\n".join(part for part in parts if part)
-    if isinstance(message, dict):
-        content = message.get("content")
-        if isinstance(content, str):
-            return content
-        if isinstance(content, list):
-            parts = []
-            for item in content:
-                text = extract_message_text(item)
-                if text:
-                    parts.append(text)
-            return "\n".join(parts)
-        if "text" in message and isinstance(message["text"], str):
-            return message["text"]
-    return str(message)
-
-
-def _parse_actions_from_json_candidate(candidate: Any) -> tuple[list[str], str]:
-    actions: list[str] = []
-    reasoning = ""
-    if isinstance(candidate, dict):
-        potential_actions = candidate.get("actions")
-        if isinstance(potential_actions, list):
-            actions = [str(a).strip() for a in potential_actions if str(a).strip()]
-        elif isinstance(potential_actions, str):
-            actions = [a.strip() for a in potential_actions.split(";") if a.strip()]
-        if "reasoning" in candidate and isinstance(candidate["reasoning"], str):
-            reasoning = candidate["reasoning"].strip()
-    return actions, reasoning
-
-
-def parse_tool_call_from_text(text: str) -> tuple[list[str], str]:
-    """Derive tool-call actions and reasoning from assistant text."""
-
-    text = (text or "").strip()
-    if not text:
-        return [], ""
-
-    # Attempt to interpret the entire payload as JSON
-    try:
-        data = json.loads(text)
-    except Exception:
-        data = None
-    else:
-        actions, reasoning = _parse_actions_from_json_candidate(data)
-        if actions:
-            return actions, reasoning or text
-
-    # Look for embedded JSON objects containing an "actions" field
-    json_like_matches = re.findall(r"\{[^{}]*actions[^{}]*\}", text, flags=re.IGNORECASE)
-    for fragment in json_like_matches:
-        try:
-            data = json.loads(fragment)
-        except Exception:
-            continue
-        actions, reasoning = _parse_actions_from_json_candidate(data)
-        if actions:
-            return actions, reasoning or text
-
-    # Patterns like "Actions: move_right, jump"
-    m = re.search(r"actions?\s*:\s*([^\n]+)", text, flags=re.IGNORECASE)
-    if m:
-        items = [part.strip() for part in m.group(1).split(",") if part.strip()]
-        if items:
-            reasoning = text[: m.start()].strip()
-            return items, reasoning
-
-    # Patterns like "Action 1: move_right"
-    actions: list[str] = []
-    reasoning_lines: list[str] = []
-    for line in text.splitlines():
-        stripped = line.strip()
-        if not stripped:
-            continue
-        match = re.match(r"action\s*\d*\s*[:\-]\s*(.+)", stripped, flags=re.IGNORECASE)
-        if match:
-            candidate = match.group(1).strip()
-            if candidate:
-                actions.append(candidate)
-        else:
-            reasoning_lines.append(stripped)
-    if actions:
-        return actions, "\n".join(reasoning_lines).strip()
-
-    return [], text
-
-
-def synthesize_tool_call_if_missing(
-    openai_response: dict[str, Any],
-    fallback_tool_name: str = "interact"
-) -> dict[str, Any]:
-    """Ensure the first choice carries a tool_call derived from text if absent.
-    
-    This is a fallback for models that don't properly support tool calling.
-    Task apps can specify their preferred fallback tool name (e.g., "interact", "execute_sequence").
-    
-    DEPRECATED: Task apps should prefer models with native tool calling support.
-    This function will be removed in a future version.
-    """
-
-    if not isinstance(openai_response, dict):
-        return openai_response
-    choices = openai_response.get("choices")
-    if not isinstance(choices, list) or not choices:
-        return openai_response
-    first = choices[0]
-    if not isinstance(first, dict):
-        return openai_response
-    message = first.get("message")
-    if not isinstance(message, dict):
-        return openai_response
-    tool_calls = message.get("tool_calls")
-    if isinstance(tool_calls, list) and tool_calls:
-        return openai_response
-
-    text = extract_message_text(message)
-    actions, reasoning = parse_tool_call_from_text(text)
-    if not actions:
-        return openai_response
-
-    # Build a fallback tool call using the provided tool name
-    payload = {
-        "actions": [str(a).strip() for a in actions if str(a).strip()],
-    }
-    if reasoning.strip():
-        payload["reasoning"] = reasoning.strip()
-    
-    tool_call = {
-        "id": f"tool_{fallback_tool_name}_fallback",
-        "type": "function",
-        "function": {
-            "name": fallback_tool_name,
-            "arguments": json.dumps(payload, ensure_ascii=False),
-        },
-    }
-
-    new_message = copy.deepcopy(message)
-    new_message["tool_calls"] = [tool_call]
-    if "content" not in new_message:
-        new_message["content"] = None
-
-    new_first = copy.deepcopy(first)
-    new_first["message"] = new_message
-    new_choices = [new_first] + choices[1:]
-
-    result = copy.deepcopy(openai_response)
-    result["choices"] = new_choices
-    return result

synth_ai/task/rubrics/__init__.py

@@ -1,56 +0,0 @@
-"""Rubric schema, loading, and scoring helpers for Task Apps.
-
-This module provides:
-- Flexible rubric models (Criterion, Rubric) for general task app use
-- Strict validators (StrictCriterion, StrictRubric) for step-wise judges
-- Loading utilities supporting JSON, YAML, and HTTP sources
-- Blending utilities for composing rubrics
-- Scoring utilities for events and outcomes
-"""
-
-# Core models (flexible validation)
-from .models import Criterion, Rubric
-
-# Loading and blending
-from .loaders import blend_rubrics, load_rubric
-
-# Scoring
-from .scoring import score_events_against_rubric, score_outcome_against_rubric
-
-# Strict validators (for judge configs)
-from .strict import (
-    StrictCriterion,
-    StrictRubric,
-    ValidationError,
-    validate_rubric_dict,
-    validate_rubric_file,
-    validate_rubric_files,
-)
-
-__all__ = [
-    # Flexible models
-    "Criterion",
-    "Rubric",
-    # Loaders
-    "load_rubric",
-    "blend_rubrics",
-    # Scoring
-    "score_events_against_rubric",
-    "score_outcome_against_rubric",
-    # Strict validators
-    "StrictCriterion",
-    "StrictRubric",
-    "ValidationError",
-    "validate_rubric_dict",
-    "validate_rubric_file",
-    "validate_rubric_files",
-]
-
-# Maintain backwards compatibility
-# Old code may import these names expecting the flexible variants
-RubricCriterion = StrictCriterion
-RubricSpec = StrictRubric
-
-
-
-