synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/task/proxy.py

@@ -1,38 +1,14 @@
-from __future__ import annotations
+"""Shared helpers for Task App proxy endpoints (OpenAI, Groq, etc.).
+
+The proxy is tool-agnostic - each task app provides its own tools schema.
+"""
 
-"""Shared helpers for Task App proxy endpoints (OpenAI, Groq, etc.)."""
+from __future__ import annotations
 
 import copy
 import json
 import re
-from typing import Any, Iterable, List, Tuple
-
-
-INTERACT_TOOL_SCHEMA: List[dict[str, Any]] = [
-    {
-        "type": "function",
-        "function": {
-            "name": "interact",
-            "description": "Perform one or more environment actions.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "actions": {
-                        "type": "array",
-                        "items": {"type": "string"},
-                        "description": "List of environment actions to execute in order.",
-                    },
-                    "reasoning": {
-                        "type": "string",
-                        "description": "Optional reasoning for the chosen actions.",
-                    },
-                },
-                "required": ["actions"],
-                "additionalProperties": False,
-            },
-        },
-    }
-]
+from typing import Any
 
 _REMOVE_FIELDS = {
     "stop_after_tool_calls",
@@ -44,14 +20,12 @@ _REMOVE_SAMPLING_FIELDS = {"temperature", "top_p"}
 _GPT5_MIN_COMPLETION_TOKENS = 16000
 
 
-def _ensure_tools(payload: dict[str, Any]) -> None:
-    tools = payload.get("tools")
-    if not isinstance(tools, list) or not tools:
-        payload["tools"] = copy.deepcopy(INTERACT_TOOL_SCHEMA)
-
-
 def prepare_for_openai(model: str | None, payload: dict[str, Any]) -> dict[str, Any]:
-    """Sanitise an OpenAI chat completions payload for Task App usage."""
+    """Sanitise an OpenAI chat completions payload for Task App usage.
+    
+    The task app is responsible for providing tools in the payload.
+    This function only handles model-specific parameter normalization.
+    """
 
     sanitized = copy.deepcopy(payload)
     for field in _REMOVE_FIELDS:
@@ -68,10 +42,18 @@ def prepare_for_openai(model: str | None, payload: dict[str, Any]) -> dict[str,
         mct = sanitized.get("max_completion_tokens")
         if not isinstance(mct, int) or mct < _GPT5_MIN_COMPLETION_TOKENS:
             sanitized["max_completion_tokens"] = _GPT5_MIN_COMPLETION_TOKENS
-        sanitized["tool_choice"] = {"type": "function", "function": {"name": "interact"}}
+        
+        # Set tool_choice to first provided tool (task app must provide tools)
+        # If tool_choice not already set and tools are provided, use the first one
+        if "tool_choice" not in sanitized:
+            tools = sanitized.get("tools", [])
+            if isinstance(tools, list) and tools:
+                first_func = tools[0].get("function", {})
+                if isinstance(first_func, dict) and "name" in first_func:
+                    sanitized["tool_choice"] = {"type": "function", "function": {"name": first_func["name"]}}
+        
         sanitized["parallel_tool_calls"] = False
 
-    _ensure_tools(sanitized)
     return sanitized
 
 
@@ -80,9 +62,13 @@ def prepare_for_groq(model: str | None, payload: dict[str, Any]) -> dict[str, An
 
     sanitized = prepare_for_openai(model, payload)
     # Groq supports `max_tokens`; prefer their native parameter when present
-    if model and "gpt-5" not in (model or ""):
-        if "max_completion_tokens" in sanitized and "max_tokens" not in payload:
-            sanitized["max_tokens"] = sanitized.pop("max_completion_tokens")
+    if (
+        model
+        and "gpt-5" not in model
+        and "max_completion_tokens" in sanitized
+        and "max_tokens" not in payload
+    ):
+        sanitized["max_tokens"] = sanitized.pop("max_completion_tokens")
     return sanitized
 
 
@@ -146,7 +132,7 @@ def _parse_actions_from_json_candidate(candidate: Any) -> tuple[list[str], str]:
     return actions, reasoning
 
 
-def parse_tool_call_from_text(text: str) -> Tuple[list[str], str]:
+def parse_tool_call_from_text(text: str) -> tuple[list[str], str]:
     """Derive tool-call actions and reasoning from assistant text."""
 
     text = (text or "").strip()
@@ -179,7 +165,7 @@ def parse_tool_call_from_text(text: str) -> Tuple[list[str], str]:
     if m:
         items = [part.strip() for part in m.group(1).split(",") if part.strip()]
         if items:
-            reasoning = text[:m.start()].strip()
+            reasoning = text[: m.start()].strip()
             return items, reasoning
 
     # Patterns like "Action 1: move_right"
@@ -202,24 +188,18 @@ def parse_tool_call_from_text(text: str) -> Tuple[list[str], str]:
     return [], text
 
 
-def _build_tool_call(actions: Iterable[str], reasoning: str) -> dict[str, Any]:
-    payload = {
-        "actions": [str(a).strip() for a in actions if str(a).strip()],
-    }
-    if reasoning.strip():
-        payload["reasoning"] = reasoning.strip()
-    return {
-        "id": "tool_interact_fallback",
-        "type": "function",
-        "function": {
-            "name": INTERACT_TOOL_SCHEMA[0]["function"]["name"],
-            "arguments": json.dumps(payload, ensure_ascii=False),
-        },
-    }
-
-
-def synthesize_tool_call_if_missing(openai_response: dict[str, Any]) -> dict[str, Any]:
-    """Ensure the first choice carries a tool_call derived from text if absent."""
+def synthesize_tool_call_if_missing(
+    openai_response: dict[str, Any],
+    fallback_tool_name: str = "interact"
+) -> dict[str, Any]:
+    """Ensure the first choice carries a tool_call derived from text if absent.
+    
+    This is a fallback for models that don't properly support tool calling.
+    Task apps can specify their preferred fallback tool name (e.g., "interact", "execute_sequence").
+    
+    DEPRECATED: Task apps should prefer models with native tool calling support.
+    This function will be removed in a future version.
+    """
 
     if not isinstance(openai_response, dict):
         return openai_response
@@ -241,10 +221,24 @@ def synthesize_tool_call_if_missing(openai_response: dict[str, Any]) -> dict[str
     if not actions:
         return openai_response
 
+    # Build a fallback tool call using the provided tool name
+    payload = {
+        "actions": [str(a).strip() for a in actions if str(a).strip()],
+    }
+    if reasoning.strip():
+        payload["reasoning"] = reasoning.strip()
+    
+    tool_call = {
+        "id": f"tool_{fallback_tool_name}_fallback",
+        "type": "function",
+        "function": {
+            "name": fallback_tool_name,
+            "arguments": json.dumps(payload, ensure_ascii=False),
+        },
+    }
+
     new_message = copy.deepcopy(message)
-    new_message["tool_calls"] = [
-        _build_tool_call(actions, reasoning)
-    ]
+    new_message["tool_calls"] = [tool_call]
     if "content" not in new_message:
         new_message["content"] = None
 
@@ -255,4 +249,3 @@ def synthesize_tool_call_if_missing(openai_response: dict[str, Any]) -> dict[str
     result = copy.deepcopy(openai_response)
     result["choices"] = new_choices
     return result
-

synth_ai/task/rubrics/__init__.py

@@ -0,0 +1,55 @@
+"""Rubric schema, loading, and scoring helpers for Task Apps.
+
+This module provides:
+- Flexible rubric models (Criterion, Rubric) for general task app use
+- Strict validators (StrictCriterion, StrictRubric) for step-wise judges
+- Loading utilities supporting JSON, YAML, and HTTP sources
+- Blending utilities for composing rubrics
+- Scoring utilities for events and outcomes
+"""
+
+# Core models (flexible validation)
+# Loading and blending
+from .loaders import blend_rubrics, load_rubric
+from .models import Criterion, Rubric
+
+# Scoring
+from .scoring import score_events_against_rubric, score_outcome_against_rubric
+
+# Strict validators (for judge configs)
+from .strict import (
+    StrictCriterion,
+    StrictRubric,
+    ValidationError,
+    validate_rubric_dict,
+    validate_rubric_file,
+    validate_rubric_files,
+)
+
+__all__ = [
+    # Flexible models
+    "Criterion",
+    "Rubric",
+    # Loaders
+    "load_rubric",
+    "blend_rubrics",
+    # Scoring
+    "score_events_against_rubric",
+    "score_outcome_against_rubric",
+    # Strict validators
+    "StrictCriterion",
+    "StrictRubric",
+    "ValidationError",
+    "validate_rubric_dict",
+    "validate_rubric_file",
+    "validate_rubric_files",
+]
+
+# Maintain backwards compatibility
+# Old code may import these names expecting the flexible variants
+RubricCriterion = StrictCriterion
+RubricSpec = StrictRubric
+
+
+
+

synth_ai/task/rubrics/loaders.py

@@ -0,0 +1,156 @@
+"""Rubric loading and blending utilities."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .models import Criterion, Rubric
+
+
+def _load_text(source: str) -> tuple[str, str | None]:
+    """Load text from file path or return as-is."""
+    path = Path(source)
+    if path.exists():
+        return path.read_text(encoding="utf-8"), path.suffix.lower()
+    return source, None
+
+
+def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
+    """Parse JSON or YAML text into a dictionary."""
+    text = text.strip()
+    if not text:
+        raise ValueError("Rubric source is empty")
+    if suffix in (".yaml", ".yml"):
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric YAML must produce a mapping") from None
+        return data
+    if text.startswith("{"):
+        return json.loads(text)
+    if text.startswith("http://") or text.startswith("https://"):
+        import requests  # type: ignore
+
+        response = requests.get(text, timeout=15)
+        response.raise_for_status()
+        return _parse_structured(response.text, suffix)
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load rubric text") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric text must decode to a mapping") from None
+        return data
+
+
+def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
+    """Load rubric from file path, dict, or return existing Rubric.
+    
+    Args:
+        source: File path (JSON/YAML), dict, existing Rubric, or None
+        
+    Returns:
+        Parsed Rubric instance or None if source is None
+        
+    Raises:
+        ValueError: If the rubric format is incorrect (e.g., backend judge format)
+        ValidationError: If the rubric fails schema validation
+    """
+    if source is None:
+        return None
+    if isinstance(source, Rubric):
+        return source
+    
+    # Load and parse the data
+    if isinstance(source, dict):
+        data = source
+    else:
+        text, suffix = _load_text(str(source))
+        data = _parse_structured(text, suffix)
+    
+    # Check if this looks like a backend judge rubric (wrong format)
+    if (
+        isinstance(data, dict)
+        and "event" in data
+        and "outcome" in data
+        and "version" not in data
+        and "goal_text" not in data
+        and "criteria" not in data
+    ):
+        source_hint = f" ({source})" if isinstance(source, str) else ""
+        raise ValueError(
+            f"Rubric appears to be in backend judge format (has 'event'/'outcome' keys){source_hint}. "
+            f"Task apps require rubrics with 'version', 'goal_text', and 'criteria' fields. "
+            f"Backend judge rubrics should be named '*_backend_judge.json' and loaded by judge functions."
+        )
+    
+    return Rubric.model_validate(data)
+
+
+def _merge_weights(base: Criterion, override: Criterion) -> float:
+    """Merge criterion weights from base and override rubrics."""
+    if override.weight != 1.0 and base.weight != 1.0:
+        return base.weight * override.weight
+    if override.weight != 1.0:
+        return override.weight
+    return base.weight
+
+
+def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None:
+    """Blend two rubrics by merging criteria and inheriting properties.
+    
+    Override rubric takes precedence for descriptions and settings.
+    Weights are merged multiplicatively when both are non-default.
+    
+    Args:
+        base: Base rubric providing defaults
+        override: Override rubric with specific customizations
+        
+    Returns:
+        Blended rubric or None if both inputs are None
+    """
+    if override is None and base is None:
+        return None
+    if base is None:
+        return override
+    if override is None:
+        return base
+
+    base_map = {criterion.id: criterion for criterion in base.criteria}
+    merged: list[Criterion] = []
+
+    for ov in override.criteria:
+        if ov.id in base_map:
+            existing = base_map.pop(ov.id)
+            merged.append(
+                Criterion(
+                    id=ov.id,
+                    description=ov.description or existing.description,
+                    weight=_merge_weights(existing, ov),
+                    required=ov.required if ov.required is not None else existing.required,
+                )
+            )
+        else:
+            merged.append(ov)
+
+    merged.extend(base_map.values())
+
+    aggregation = override.aggregation
+    if aggregation == "inherit":
+        aggregation = base.aggregation
+
+    return Rubric(
+        version=override.version or base.version,
+        goal_text=override.goal_text or base.goal_text,
+        criteria=merged,
+        aggregation=aggregation,
+    )

synth_ai/task/rubrics/models.py

@@ -0,0 +1,57 @@
+"""Rubric and Criterion data models."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Criterion(BaseModel):
+    """Single scoring criterion within a rubric.
+    
+    Flexible variant allowing weights > 1.0 and no normalization requirement.
+    Used by task apps for general rubric scoring.
+    """
+
+    id: str
+    description: str
+    weight: float = 1.0
+    required: bool = False
+
+    @field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if value <= 0:
+            raise ValueError("criterion weight must be positive")
+        return value
+
+
+class Rubric(BaseModel):
+    """Rubric definition for scoring task app outcomes.
+    
+    Supports flexible aggregation and blending. Criteria weights do not need
+    to sum to 1.0, making this suitable for general task app usage.
+    """
+
+    version: str
+    goal_text: str | None = None
+    criteria: list[Criterion] = Field(default_factory=list)
+    aggregation: str = "weighted_sum"
+
+    @field_validator("aggregation")
+    @classmethod
+    def _validate_aggregation(cls, value: str) -> str:
+        allowed = {"sum", "weighted_sum", "custom", "inherit"}
+        if value not in allowed:
+            raise ValueError(f"aggregation must be one of {sorted(allowed)}")
+        return value
+
+    @field_validator("criteria")
+    @classmethod
+    def _validate_criteria(cls, criteria: list[Criterion]) -> list[Criterion]:
+        seen = set()
+        for criterion in criteria:
+            if criterion.id in seen:
+                raise ValueError(f"duplicate criterion id: {criterion.id}")
+            seen.add(criterion.id)
+        return criteria
+

synth_ai/task/rubrics/scoring.py

@@ -0,0 +1,116 @@
+"""Rubric scoring utilities for events and outcomes."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+from .models import Criterion, Rubric
+
+
+def _as_float(value: Any) -> float | None:
+    """Safely convert value to float, returning None on failure."""
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _score(
+    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
+) -> dict[str, Any]:
+    """Compute aggregate score from criterion values.
+    
+    Args:
+        criteria: List of criteria defining scoring dimensions
+        values: Map of criterion IDs to scores
+        aggregation: How to aggregate ("sum", "weighted_sum", "custom")
+        
+    Returns:
+        Dict with aggregation method, total score, and per-criterion breakdown
+    """
+    if aggregation == "inherit":
+        aggregation = "weighted_sum"
+    per_criterion: dict[str, dict[str, Any]] = {}
+    total = 0.0
+    total_weight = 0.0
+    for criterion in criteria:
+        score = values.get(criterion.id, 0.0)
+        per_criterion[criterion.id] = {
+            "score": score,
+            "weight": criterion.weight,
+            "required": criterion.required,
+        }
+        if aggregation == "sum":
+            total += score
+        elif aggregation == "weighted_sum":
+            total += score * criterion.weight
+            total_weight += criterion.weight
+    if aggregation == "weighted_sum" and total_weight > 0:
+        total = total / total_weight
+    if aggregation == "custom":
+        total = None  # type: ignore[assignment]
+    return {
+        "aggregation": aggregation,
+        "score": total,
+        "per_criterion": per_criterion,
+    }
+
+
+def score_events_against_rubric(
+    events: list[dict[str, Any]], rubric: Rubric | None
+) -> dict[str, Any]:
+    """Score a list of evaluation events against a rubric.
+    
+    Events should contain criterion_id/id/criterion and score fields.
+    
+    Args:
+        events: List of event dicts with scoring info
+        rubric: Rubric defining criteria and aggregation
+        
+    Returns:
+        Scoring result with total and per-criterion scores
+    """
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    for event in events or []:
+        if not isinstance(event, dict):
+            continue
+        cid = event.get("criterion_id") or event.get("id") or event.get("criterion")
+        score = _as_float(event.get("score"))
+        if cid and score is not None:
+            values[str(cid)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
+    """Score a rollout outcome against a rubric.
+    
+    Outcome should be a dict mapping criterion IDs to scores, optionally
+    nested under a "criteria" key.
+    
+    Args:
+        outcome: Outcome dict with criterion scores
+        rubric: Rubric defining criteria and aggregation
+        
+    Returns:
+        Scoring result with total and per-criterion scores
+    """
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    if isinstance(outcome, dict):
+        candidates = (
+            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        )
+        if isinstance(candidates, dict):
+            for key, value in candidates.items():
+                score = _as_float(value)
+                if score is not None:
+                    values[str(key)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+
+