synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/task/proxy.py

@@ -0,0 +1,287 @@
+"""Shared helpers for Task App proxy endpoints (OpenAI, Groq, etc.).
+
+The proxy is tool-agnostic - each task app provides its own tools schema.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import re
+from typing import Any
+
+_REMOVE_FIELDS = {
+    "stop_after_tool_calls",
+    "thinking_mode",
+    "thinking_budget",
+    "reasoning",
+}
+_REMOVE_SAMPLING_FIELDS = {"temperature", "top_p"}
+_GPT5_MIN_COMPLETION_TOKENS = 16000
+
+
+def prepare_for_openai(model: str | None, payload: dict[str, Any]) -> dict[str, Any]:
+    """Sanitise an OpenAI chat completions payload for Task App usage.
+    
+    The task app is responsible for providing tools in the payload.
+    This function only handles model-specific parameter normalization.
+    """
+
+    sanitized = copy.deepcopy(payload)
+    for field in _REMOVE_FIELDS:
+        sanitized.pop(field, None)
+
+    if model and "gpt-5" in model:
+        max_tokens = sanitized.pop("max_tokens", None)
+        if "max_completion_tokens" not in sanitized and isinstance(max_tokens, int):
+            sanitized["max_completion_tokens"] = max_tokens
+        elif max_tokens is not None:
+            sanitized.setdefault("max_completion_tokens", max_tokens)
+        for field in _REMOVE_SAMPLING_FIELDS:
+            sanitized.pop(field, None)
+        mct = sanitized.get("max_completion_tokens")
+        if not isinstance(mct, int) or mct < _GPT5_MIN_COMPLETION_TOKENS:
+            sanitized["max_completion_tokens"] = _GPT5_MIN_COMPLETION_TOKENS
+        
+        # Set tool_choice to first provided tool (task app must provide tools)
+        # If tool_choice not already set and tools are provided, use the first one
+        if "tool_choice" not in sanitized:
+            tools = sanitized.get("tools", [])
+            if isinstance(tools, list) and tools:
+                first_func = tools[0].get("function", {})
+                if isinstance(first_func, dict) and "name" in first_func:
+                    sanitized["tool_choice"] = {"type": "function", "function": {"name": first_func["name"]}}
+        
+        sanitized["parallel_tool_calls"] = False
+
+    return sanitized
+
+
+def prepare_for_groq(model: str | None, payload: dict[str, Any]) -> dict[str, Any]:
+    """Groq uses the OpenAI schema; reuse most normalisation rules."""
+
+    sanitized = prepare_for_openai(model, payload)
+    # Groq supports `max_tokens`; prefer their native parameter when present
+    if (
+        model
+        and "gpt-5" not in model
+        and "max_completion_tokens" in sanitized
+        and "max_tokens" not in payload
+    ):
+        sanitized["max_tokens"] = sanitized.pop("max_completion_tokens")
+
+    # Normalize response_format for Groq
+    # Groq supports json_schema on newer models (llama-3.3-70b, etc.)
+    # For older models, fall back to json_object
+    normalize_response_format_for_groq(model, sanitized)
+
+    return sanitized
+
+
+# Models that support json_schema response_format
+_GROQ_JSON_SCHEMA_MODELS = {
+    "llama-3.3-70b",
+    "llama-3.1-70b-versatile",
+    "llama-70b",
+    "mixtral-8x7b",
+}
+
+
+def normalize_response_format_for_groq(model: str | None, payload: dict[str, Any]) -> None:
+    """Normalize response_format for Groq provider compatibility.
+
+    Groq supports json_schema on newer models (llama-3.3-70b, etc.)
+    For older models, fall back to json_object.
+    """
+    response_format = payload.get("response_format")
+    if not response_format or not isinstance(response_format, dict):
+        return
+
+    model_lower = (model or "").lower()
+    supports_json_schema = any(
+        supported in model_lower
+        for supported in _GROQ_JSON_SCHEMA_MODELS
+    )
+
+    # If model doesn't support json_schema and we're using it, convert to json_object
+    if not supports_json_schema and response_format.get("type") == "json_schema":
+        payload["response_format"] = {"type": "json_object"}
+
+
+def inject_system_hint(payload: dict[str, Any], hint: str) -> dict[str, Any]:
+    """Insert or augment a system message with the provided hint (idempotent)."""
+
+    if not hint:
+        return payload
+    cloned = copy.deepcopy(payload)
+    messages = cloned.get("messages")
+    if not isinstance(messages, list):
+        return cloned
+    if messages and isinstance(messages[0], dict) and messages[0].get("role") == "system":
+        content = messages[0].get("content")
+        if isinstance(content, str) and hint not in content:
+            messages[0] = dict(messages[0])
+            messages[0]["content"] = content.rstrip() + ("\n\n" if content else "") + hint
+    else:
+        messages.insert(0, {"role": "system", "content": hint})
+    cloned["messages"] = messages
+    return cloned
+
+
+def extract_message_text(message: Any) -> str:
+    """Return best-effort text from an OpenAI-style message structure."""
+
+    if message is None:
+        return ""
+    if isinstance(message, str):
+        return message
+    if isinstance(message, list):
+        parts = [extract_message_text(part) for part in message]
+        return "\n".join(part for part in parts if part)
+    if isinstance(message, dict):
+        content = message.get("content")
+        if isinstance(content, str):
+            return content
+        if isinstance(content, list):
+            parts = []
+            for item in content:
+                text = extract_message_text(item)
+                if text:
+                    parts.append(text)
+            return "\n".join(parts)
+        if "text" in message and isinstance(message["text"], str):
+            return message["text"]
+    return str(message)
+
+
+def _parse_actions_from_json_candidate(candidate: Any) -> tuple[list[str], str]:
+    actions: list[str] = []
+    reasoning = ""
+    if isinstance(candidate, dict):
+        potential_actions = candidate.get("actions")
+        if isinstance(potential_actions, list):
+            actions = [str(a).strip() for a in potential_actions if str(a).strip()]
+        elif isinstance(potential_actions, str):
+            actions = [a.strip() for a in potential_actions.split(";") if a.strip()]
+        if "reasoning" in candidate and isinstance(candidate["reasoning"], str):
+            reasoning = candidate["reasoning"].strip()
+    return actions, reasoning
+
+
+def parse_tool_call_from_text(text: str) -> tuple[list[str], str]:
+    """Derive tool-call actions and reasoning from assistant text."""
+
+    text = (text or "").strip()
+    if not text:
+        return [], ""
+
+    # Attempt to interpret the entire payload as JSON
+    try:
+        data = json.loads(text)
+    except Exception:
+        data = None
+    else:
+        actions, reasoning = _parse_actions_from_json_candidate(data)
+        if actions:
+            return actions, reasoning or text
+
+    # Look for embedded JSON objects containing an "actions" field
+    json_like_matches = re.findall(r"\{[^{}]*actions[^{}]*\}", text, flags=re.IGNORECASE)
+    for fragment in json_like_matches:
+        try:
+            data = json.loads(fragment)
+        except Exception:
+            continue
+        actions, reasoning = _parse_actions_from_json_candidate(data)
+        if actions:
+            return actions, reasoning or text
+
+    # Patterns like "Actions: move_right, jump"
+    m = re.search(r"actions?\s*:\s*([^\n]+)", text, flags=re.IGNORECASE)
+    if m:
+        items = [part.strip() for part in m.group(1).split(",") if part.strip()]
+        if items:
+            reasoning = text[: m.start()].strip()
+            return items, reasoning
+
+    # Patterns like "Action 1: move_right"
+    actions: list[str] = []
+    reasoning_lines: list[str] = []
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped:
+            continue
+        match = re.match(r"action\s*\d*\s*[:\-]\s*(.+)", stripped, flags=re.IGNORECASE)
+        if match:
+            candidate = match.group(1).strip()
+            if candidate:
+                actions.append(candidate)
+        else:
+            reasoning_lines.append(stripped)
+    if actions:
+        return actions, "\n".join(reasoning_lines).strip()
+
+    return [], text
+
+
+def synthesize_tool_call_if_missing(
+    openai_response: dict[str, Any],
+    fallback_tool_name: str = "interact"
+) -> dict[str, Any]:
+    """Ensure the first choice carries a tool_call derived from text if absent.
+    
+    This is a fallback for models that don't properly support tool calling.
+    Task apps can specify their preferred fallback tool name (e.g., "interact", "execute_sequence").
+    
+    DEPRECATED: Task apps should prefer models with native tool calling support.
+    This function will be removed in a future version.
+    """
+
+    if not isinstance(openai_response, dict):
+        return openai_response
+    choices = openai_response.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return openai_response
+    first = choices[0]
+    if not isinstance(first, dict):
+        return openai_response
+    message = first.get("message")
+    if not isinstance(message, dict):
+        return openai_response
+    tool_calls = message.get("tool_calls")
+    if isinstance(tool_calls, list) and tool_calls:
+        return openai_response
+
+    text = extract_message_text(message)
+    actions, reasoning = parse_tool_call_from_text(text)
+    if not actions:
+        return openai_response
+
+    # Build a fallback tool call using the provided tool name
+    payload = {
+        "actions": [str(a).strip() for a in actions if str(a).strip()],
+    }
+    if reasoning.strip():
+        payload["reasoning"] = reasoning.strip()
+    
+    tool_call = {
+        "id": f"tool_{fallback_tool_name}_fallback",
+        "type": "function",
+        "function": {
+            "name": fallback_tool_name,
+            "arguments": json.dumps(payload, ensure_ascii=False),
+        },
+    }
+
+    new_message = copy.deepcopy(message)
+    new_message["tool_calls"] = [tool_call]
+    if "content" not in new_message:
+        new_message["content"] = None
+
+    new_first = copy.deepcopy(first)
+    new_first["message"] = new_message
+    new_choices = [new_first] + choices[1:]
+
+    result = copy.deepcopy(openai_response)
+    result["choices"] = new_choices
+    return result

synth_ai/sdk/task/rubrics/__init__.py

@@ -0,0 +1,54 @@
+"""Rubric schema, loading, and scoring helpers for Task Apps.
+
+This module provides:
+- Flexible rubric models (Criterion, Rubric) for general task app use
+- Strict validators (StrictCriterion, StrictRubric) for step-wise verifiers
+- Loading utilities supporting JSON, YAML, and HTTP sources
+- Blending utilities for composing rubrics
+- Scoring utilities for events and outcomes
+"""
+
+# Core models (flexible validation)
+# Loading and blending
+from .loaders import blend_rubrics, load_rubric
+from .models import Criterion, Rubric
+
+# Scoring
+from .scoring import score_events_against_rubric, score_outcome_against_rubric
+
+# Strict validators (for verifier configs)
+from .strict import (
+    StrictCriterion,
+    StrictRubric,
+    ValidationError,
+    validate_rubric_dict,
+    validate_rubric_file,
+    validate_rubric_files,
+)
+
+__all__ = [
+    # Flexible models
+    "Criterion",
+    "Rubric",
+    # Loaders
+    "load_rubric",
+    "blend_rubrics",
+    # Scoring
+    "score_events_against_rubric",
+    "score_outcome_against_rubric",
+    # Strict validators
+    "StrictCriterion",
+    "StrictRubric",
+    "ValidationError",
+    "validate_rubric_dict",
+    "validate_rubric_file",
+    "validate_rubric_files",
+]
+
+# Maintain backwards compatibility
+# Old code may import these names expecting the flexible variants
+RubricCriterion = StrictCriterion
+RubricSpec = StrictRubric
+
+
+

synth_ai/sdk/task/rubrics/loaders.py

@@ -0,0 +1,156 @@
+"""Rubric loading and blending utilities."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .models import Criterion, Rubric
+
+
+def _load_text(source: str) -> tuple[str, str | None]:
+    """Load text from file path or return as-is."""
+    path = Path(source)
+    if path.exists():
+        return path.read_text(encoding="utf-8"), path.suffix.lower()
+    return source, None
+
+
+def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
+    """Parse JSON or YAML text into a dictionary."""
+    text = text.strip()
+    if not text:
+        raise ValueError("Rubric source is empty")
+    if suffix in (".yaml", ".yml"):
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric YAML must produce a mapping") from None
+        return data
+    if text.startswith("{"):
+        return json.loads(text)
+    if text.startswith("http://") or text.startswith("https://"):
+        import requests  # type: ignore
+
+        response = requests.get(text, timeout=15)
+        response.raise_for_status()
+        return _parse_structured(response.text, suffix)
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load rubric text") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric text must decode to a mapping") from None
+        return data
+
+
+def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
+    """Load rubric from file path, dict, or return existing Rubric.
+    
+    Args:
+        source: File path (JSON/YAML), dict, existing Rubric, or None
+        
+    Returns:
+        Parsed Rubric instance or None if source is None
+        
+    Raises:
+        ValueError: If the rubric format is incorrect (e.g., backend verifier format)
+        ValidationError: If the rubric fails schema validation
+    """
+    if source is None:
+        return None
+    if isinstance(source, Rubric):
+        return source
+    
+    # Load and parse the data
+    if isinstance(source, dict):
+        data = source
+    else:
+        text, suffix = _load_text(str(source))
+        data = _parse_structured(text, suffix)
+    
+    # Check if this looks like a backend verifier rubric (wrong format)
+    if (
+        isinstance(data, dict)
+        and "event" in data
+        and "outcome" in data
+        and "version" not in data
+        and "goal_text" not in data
+        and "criteria" not in data
+    ):
+        source_hint = f" ({source})" if isinstance(source, str) else ""
+        raise ValueError(
+            f"Rubric appears to be in backend verifier format (has 'event'/'outcome' keys){source_hint}. "
+            f"Task apps require rubrics with 'version', 'goal_text', and 'criteria' fields. "
+            f"Backend verifier rubrics should be named '*_backend_verifier.json' and loaded by verifier functions."
+        )
+    
+    return Rubric.model_validate(data)
+
+
+def _merge_weights(base: Criterion, override: Criterion) -> float:
+    """Merge criterion weights from base and override rubrics."""
+    if override.weight != 1.0 and base.weight != 1.0:
+        return base.weight * override.weight
+    if override.weight != 1.0:
+        return override.weight
+    return base.weight
+
+
+def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None:
+    """Blend two rubrics by merging criteria and inheriting properties.
+    
+    Override rubric takes precedence for descriptions and settings.
+    Weights are merged multiplicatively when both are non-default.
+    
+    Args:
+        base: Base rubric providing defaults
+        override: Override rubric with specific customizations
+        
+    Returns:
+        Blended rubric or None if both inputs are None
+    """
+    if override is None and base is None:
+        return None
+    if base is None:
+        return override
+    if override is None:
+        return base
+
+    base_map = {criterion.id: criterion for criterion in base.criteria}
+    merged: list[Criterion] = []
+
+    for ov in override.criteria:
+        if ov.id in base_map:
+            existing = base_map.pop(ov.id)
+            merged.append(
+                Criterion(
+                    id=ov.id,
+                    description=ov.description or existing.description,
+                    weight=_merge_weights(existing, ov),
+                    required=ov.required if ov.required is not None else existing.required,
+                )
+            )
+        else:
+            merged.append(ov)
+
+    merged.extend(base_map.values())
+
+    aggregation = override.aggregation
+    if aggregation == "inherit":
+        aggregation = base.aggregation
+
+    return Rubric(
+        version=override.version or base.version,
+        goal_text=override.goal_text or base.goal_text,
+        criteria=merged,
+        aggregation=aggregation,
+    )

synth_ai/sdk/task/rubrics/strict.py

@@ -0,0 +1,148 @@
+"""Strict rubric validators for step-wise verifiers.
+
+These validators enforce stricter constraints than the general-purpose rubrics:
+- Weights must be ≤ 1.0 and sum to exactly 1.0
+- Only weighted_sum aggregation is allowed
+- All required fields must be non-empty
+
+Used primarily for validation in verifier configurations.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, Literal
+
+import pydantic
+
+
+class StrictCriterion(pydantic.BaseModel):
+    """Single scoring criterion with strict validation.
+    
+    Enforces:
+    - Weight ≤ 1.0 (for proper normalization)
+    - Weight > 0.0 (positive)
+    - Non-empty strings
+    """
+
+    id: str
+    description: str
+    weight: float
+    scale: str | None = None
+
+    @pydantic.field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("weight must be a finite number")
+        if value <= 0.0:
+            raise ValueError("weight must be positive")
+        if value > 1.0:
+            raise ValueError("weight must be <= 1.0")
+        return value
+
+    @pydantic.field_validator("id", "description", mode="before")
+    @classmethod
+    def _strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        return value
+
+
+class StrictRubric(pydantic.BaseModel):
+    """Strict rubric definition for step-wise verifiers.
+    
+    Enforces:
+    - Weights must sum to 1.0
+    - Only weighted_sum aggregation
+    - Non-empty version and goal_text
+    - At least one criterion
+    """
+
+    version: str
+    goal_text: str
+    aggregation: Literal["weighted_sum"]
+    criteria: list[StrictCriterion]
+
+    @pydantic.model_validator(mode="after")
+    def _validate_weights(self) -> StrictRubric:
+        if not self.criteria:
+            raise ValueError("rubric must declare at least one criterion")
+        total_weight = sum(criterion.weight for criterion in self.criteria)
+        if not math.isclose(total_weight, 1.0, abs_tol=1e-6, rel_tol=1e-6):
+            raise ValueError(
+                f"criterion weights must sum to 1 (got {total_weight:.6f})"
+            )
+        return self
+
+    @pydantic.field_validator("version")
+    @classmethod
+    def _non_empty_version(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("version string must not be empty")
+        return value
+
+    @pydantic.field_validator("goal_text")
+    @classmethod
+    def _non_empty_goal_text(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("goal_text must not be empty")
+        return value
+
+
+# Re-export pydantic's ValidationError for convenience
+ValidationError = pydantic.ValidationError
+
+
+def validate_rubric_dict(payload: dict[str, Any]) -> StrictRubric:
+    """Validate an in-memory rubric payload with strict rules.
+    
+    Args:
+        payload: Dictionary representing the rubric JSON
+        
+    Returns:
+        Validated StrictRubric instance
+        
+    Raises:
+        ValidationError: If payload is invalid or doesn't meet strict constraints
+    """
+    if not isinstance(payload, dict):
+        raise TypeError("rubric payload must be a dictionary")
+    return StrictRubric.model_validate(payload)
+
+
+def _load_payload_from_file(path: Path) -> dict[str, Any]:
+    """Load JSON rubric from file."""
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Unsupported rubric file type: {path}")
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text)
+
+
+def validate_rubric_file(path: Path) -> StrictRubric:
+    """Load and validate a rubric file with strict rules.
+    
+    Args:
+        path: Path to a JSON rubric document
+        
+    Returns:
+        Validated StrictRubric instance
+    """
+    payload = _load_payload_from_file(path)
+    return validate_rubric_dict(payload)
+
+
+def validate_rubric_files(paths: Iterable[Path]) -> list[StrictRubric]:
+    """Validate multiple rubric files with strict rules.
+    
+    Useful for bulk validation inside tests or CI checks.
+    """
+    validated: list[StrictRubric] = []
+    for path in paths:
+        validated.append(validate_rubric_file(path))
+    return validated