synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/task/rubrics/scoring.py

@@ -0,0 +1,116 @@
+"""Rubric scoring utilities for events and outcomes."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+from .models import Criterion, Rubric
+
+
+def _as_float(value: Any) -> float | None:
+    """Safely convert value to float, returning None on failure."""
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _score(
+    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
+) -> dict[str, Any]:
+    """Compute aggregate score from criterion values.
+    
+    Args:
+        criteria: List of criteria defining scoring dimensions
+        values: Map of criterion IDs to scores
+        aggregation: How to aggregate ("sum", "weighted_sum", "custom")
+        
+    Returns:
+        Dict with aggregation method, total score, and per-criterion breakdown
+    """
+    if aggregation == "inherit":
+        aggregation = "weighted_sum"
+    per_criterion: dict[str, dict[str, Any]] = {}
+    total = 0.0
+    total_weight = 0.0
+    for criterion in criteria:
+        score = values.get(criterion.id, 0.0)
+        per_criterion[criterion.id] = {
+            "score": score,
+            "weight": criterion.weight,
+            "required": criterion.required,
+        }
+        if aggregation == "sum":
+            total += score
+        elif aggregation == "weighted_sum":
+            total += score * criterion.weight
+            total_weight += criterion.weight
+    if aggregation == "weighted_sum" and total_weight > 0:
+        total = total / total_weight
+    if aggregation == "custom":
+        total = None  # type: ignore[assignment]
+    return {
+        "aggregation": aggregation,
+        "score": total,
+        "per_criterion": per_criterion,
+    }
+
+
+def score_events_against_rubric(
+    events: list[dict[str, Any]], rubric: Rubric | None
+) -> dict[str, Any]:
+    """Score a list of evaluation events against a rubric.
+    
+    Events should contain criterion_id/id/criterion and score fields.
+    
+    Args:
+        events: List of event dicts with scoring info
+        rubric: Rubric defining criteria and aggregation
+        
+    Returns:
+        Scoring result with total and per-criterion scores
+    """
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    for event in events or []:
+        if not isinstance(event, dict):
+            continue
+        cid = event.get("criterion_id") or event.get("id") or event.get("criterion")
+        score = _as_float(event.get("score"))
+        if cid and score is not None:
+            values[str(cid)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
+    """Score a rollout outcome against a rubric.
+    
+    Outcome should be a dict mapping criterion IDs to scores, optionally
+    nested under a "criteria" key.
+    
+    Args:
+        outcome: Outcome dict with criterion scores
+        rubric: Rubric defining criteria and aggregation
+        
+    Returns:
+        Scoring result with total and per-criterion scores
+    """
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    if isinstance(outcome, dict):
+        candidates = (
+            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        )
+        if isinstance(candidates, dict):
+            for key, value in candidates.items():
+                score = _as_float(value)
+                if score is not None:
+                    values[str(key)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+
+

synth_ai/task/rubrics/strict.py

@@ -0,0 +1,149 @@
+"""Strict rubric validators for step-wise judges.
+
+These validators enforce stricter constraints than the general-purpose rubrics:
+- Weights must be ≤ 1.0 and sum to exactly 1.0
+- Only weighted_sum aggregation is allowed
+- All required fields must be non-empty
+
+Used primarily for validation in judge configurations.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, Literal
+
+import pydantic
+
+
+class StrictCriterion(pydantic.BaseModel):
+    """Single scoring criterion with strict validation.
+    
+    Enforces:
+    - Weight ≤ 1.0 (for proper normalization)
+    - Weight > 0.0 (positive)
+    - Non-empty strings
+    """
+
+    id: str
+    description: str
+    weight: float
+    scale: str | None = None
+
+    @pydantic.field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("weight must be a finite number")
+        if value <= 0.0:
+            raise ValueError("weight must be positive")
+        if value > 1.0:
+            raise ValueError("weight must be <= 1.0")
+        return value
+
+    @pydantic.field_validator("id", "description", mode="before")
+    @classmethod
+    def _strip_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        return value
+
+
+class StrictRubric(pydantic.BaseModel):
+    """Strict rubric definition for step-wise judges.
+    
+    Enforces:
+    - Weights must sum to 1.0
+    - Only weighted_sum aggregation
+    - Non-empty version and goal_text
+    - At least one criterion
+    """
+
+    version: str
+    goal_text: str
+    aggregation: Literal["weighted_sum"]
+    criteria: list[StrictCriterion]
+
+    @pydantic.model_validator(mode="after")
+    def _validate_weights(self) -> StrictRubric:
+        if not self.criteria:
+            raise ValueError("rubric must declare at least one criterion")
+        total_weight = sum(criterion.weight for criterion in self.criteria)
+        if not math.isclose(total_weight, 1.0, abs_tol=1e-6, rel_tol=1e-6):
+            raise ValueError(
+                f"criterion weights must sum to 1 (got {total_weight:.6f})"
+            )
+        return self
+
+    @pydantic.field_validator("version")
+    @classmethod
+    def _non_empty_version(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("version string must not be empty")
+        return value
+
+    @pydantic.field_validator("goal_text")
+    @classmethod
+    def _non_empty_goal_text(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("goal_text must not be empty")
+        return value
+
+
+# Re-export pydantic's ValidationError for convenience
+ValidationError = pydantic.ValidationError
+
+
+def validate_rubric_dict(payload: dict[str, Any]) -> StrictRubric:
+    """Validate an in-memory rubric payload with strict rules.
+    
+    Args:
+        payload: Dictionary representing the rubric JSON
+        
+    Returns:
+        Validated StrictRubric instance
+        
+    Raises:
+        ValidationError: If payload is invalid or doesn't meet strict constraints
+    """
+    if not isinstance(payload, dict):
+        raise TypeError("rubric payload must be a dictionary")
+    return StrictRubric.model_validate(payload)
+
+
+def _load_payload_from_file(path: Path) -> dict[str, Any]:
+    """Load JSON rubric from file."""
+    if path.suffix.lower() != ".json":
+        raise ValueError(f"Unsupported rubric file type: {path}")
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text)
+
+
+def validate_rubric_file(path: Path) -> StrictRubric:
+    """Load and validate a rubric file with strict rules.
+    
+    Args:
+        path: Path to a JSON rubric document
+        
+    Returns:
+        Validated StrictRubric instance
+    """
+    payload = _load_payload_from_file(path)
+    return validate_rubric_dict(payload)
+
+
+def validate_rubric_files(paths: Iterable[Path]) -> list[StrictRubric]:
+    """Validate multiple rubric files with strict rules.
+    
+    Useful for bulk validation inside tests or CI checks.
+    """
+    validated: list[StrictRubric] = []
+    for path in paths:
+        validated.append(validate_rubric_file(path))
+    return validated
+

synth_ai/task/rubrics.py

@@ -0,0 +1,219 @@
+"""Rubric schema, loading, and scoring helpers for Task Apps."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Criterion(BaseModel):
+    id: str
+    description: str
+    weight: float = 1.0
+    required: bool = False
+
+    @field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if value <= 0:
+            raise ValueError("criterion weight must be positive")
+        return value
+
+
+class Rubric(BaseModel):
+    version: str
+    goal_text: str | None = None
+    criteria: list[Criterion] = Field(default_factory=list)
+    aggregation: str = "weighted_sum"
+
+    @field_validator("aggregation")
+    @classmethod
+    def _validate_aggregation(cls, value: str) -> str:
+        allowed = {"sum", "weighted_sum", "custom", "inherit"}
+        if value not in allowed:
+            raise ValueError(f"aggregation must be one of {sorted(allowed)}")
+        return value
+
+    @field_validator("criteria")
+    @classmethod
+    def _validate_criteria(cls, criteria: list[Criterion]) -> list[Criterion]:
+        seen = set()
+        for criterion in criteria:
+            if criterion.id in seen:
+                raise ValueError(f"duplicate criterion id: {criterion.id}")
+            seen.add(criterion.id)
+        return criteria
+
+
+def _load_text(source: str) -> tuple[str, str | None]:
+    path = Path(source)
+    if path.exists():
+        return path.read_text(encoding="utf-8"), path.suffix.lower()
+    return source, None
+
+
+def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
+    text = text.strip()
+    if not text:
+        raise ValueError("Rubric source is empty")
+    if suffix in (".yaml", ".yml"):
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric YAML must produce a mapping") from None
+        return data
+    if text.startswith("{"):
+        return json.loads(text)
+    if text.startswith("http://") or text.startswith("https://"):
+        import requests  # type: ignore
+
+        response = requests.get(text, timeout=15)
+        response.raise_for_status()
+        return _parse_structured(response.text, suffix)
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load rubric text") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric text must decode to a mapping") from None
+        return data
+
+
+def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
+    if source is None:
+        return None
+    if isinstance(source, Rubric):
+        return source
+    if isinstance(source, dict):
+        return Rubric.model_validate(source)
+    text, suffix = _load_text(str(source))
+    data = _parse_structured(text, suffix)
+    return Rubric.model_validate(data)
+
+
+def _merge_weights(base: Criterion, override: Criterion) -> float:
+    if override.weight != 1.0 and base.weight != 1.0:
+        return base.weight * override.weight
+    if override.weight != 1.0:
+        return override.weight
+    return base.weight
+
+
+def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None:
+    if override is None and base is None:
+        return None
+    if base is None:
+        return override
+    if override is None:
+        return base
+
+    base_map = {criterion.id: criterion for criterion in base.criteria}
+    merged: list[Criterion] = []
+
+    for ov in override.criteria:
+        if ov.id in base_map:
+            existing = base_map.pop(ov.id)
+            merged.append(
+                Criterion(
+                    id=ov.id,
+                    description=ov.description or existing.description,
+                    weight=_merge_weights(existing, ov),
+                    required=ov.required if ov.required is not None else existing.required,
+                )
+            )
+        else:
+            merged.append(ov)
+
+    merged.extend(base_map.values())
+
+    aggregation = override.aggregation
+    if aggregation == "inherit":
+        aggregation = base.aggregation
+
+    return Rubric(
+        version=override.version or base.version,
+        goal_text=override.goal_text or base.goal_text,
+        criteria=merged,
+        aggregation=aggregation,
+    )
+
+
+def _as_float(value: Any) -> float | None:
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _score(
+    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
+) -> dict[str, Any]:
+    if aggregation == "inherit":
+        aggregation = "weighted_sum"
+    per_criterion: dict[str, dict[str, Any]] = {}
+    total = 0.0
+    total_weight = 0.0
+    for criterion in criteria:
+        score = values.get(criterion.id, 0.0)
+        per_criterion[criterion.id] = {
+            "score": score,
+            "weight": criterion.weight,
+            "required": criterion.required,
+        }
+        if aggregation == "sum":
+            total += score
+        elif aggregation == "weighted_sum":
+            total += score * criterion.weight
+            total_weight += criterion.weight
+    if aggregation == "weighted_sum" and total_weight > 0:
+        total = total / total_weight
+    if aggregation == "custom":
+        total = None  # type: ignore[assignment]
+    return {
+        "aggregation": aggregation,
+        "score": total,
+        "per_criterion": per_criterion,
+    }
+
+
+def score_events_against_rubric(
+    events: list[dict[str, Any]], rubric: Rubric | None
+) -> dict[str, Any]:
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    for event in events or []:
+        if not isinstance(event, dict):
+            continue
+        cid = event.get("criterion_id") or event.get("id") or event.get("criterion")
+        score = _as_float(event.get("score"))
+        if cid and score is not None:
+            values[str(cid)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    if isinstance(outcome, dict):
+        candidates = (
+            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        )
+        if isinstance(candidates, dict):
+            for key, value in candidates.items():
+                score = _as_float(value)
+                if score is not None:
+                    values[str(key)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)