synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

synth_ai/task/config.py

@@ -0,0 +1,259 @@
+"""Configuration dataclasses for task app CLI commands (eval, filter)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass(slots=True)
+class EvalConfig:
+    """Configuration for 'synth-ai eval' command.
+    
+    Validates and provides defaults for evaluation runs against task apps.
+    """
+    
+    # Required: Task app identifier
+    app_id: str
+    
+    # Required: Model to evaluate
+    model: str
+    
+    # Required: Seeds to run
+    seeds: list[int]
+    
+    # Optional: Task app URL (None = spawn in-process)
+    task_app_url: str | None = None
+    
+    # Optional: Data split to use
+    split: str = "train"
+    
+    # Optional: Maximum turns/steps per episode
+    max_turns: int | None = None
+    
+    # Optional: Maximum LLM calls per episode
+    max_llm_calls: int = 10
+    
+    # Optional: Concurrency for parallel rollouts
+    concurrency: int = 1
+    
+    # Optional: Environment name
+    env_name: str | None = None
+    
+    # Optional: Policy name
+    policy_name: str | None = None
+    
+    # Optional: Trace format ("compact", "full", "structured")
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+    
+    # Optional: Whether to return traces in response
+    return_trace: bool = False
+    
+    # Optional: Operations sequence (if not provided, generates default)
+    ops: list[str] | None = None
+    
+    # Optional: Environment config overrides
+    env_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Policy config overrides
+    policy_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for traces
+    metadata: dict[str, str] = field(default_factory=dict)
+    
+    # Optional: SQL query for metadata filtering
+    metadata_sql: str | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.app_id:
+            raise ValueError("app_id is required")
+        
+        if not self.model:
+            raise ValueError("model is required")
+        
+        if not self.seeds:
+            raise ValueError("seeds list cannot be empty")
+        
+        if not isinstance(self.seeds, list):
+            raise ValueError("seeds must be a list of integers")
+        
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        
+        if self.max_llm_calls < 1:
+            raise ValueError("max_llm_calls must be >= 1")
+        
+        if self.max_turns is not None and self.max_turns < 1:
+            raise ValueError("max_turns must be >= 1")
+        
+        if self.trace_format not in ("compact", "full", "structured"):
+            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
+        """Create EvalConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with eval configuration
+            
+        Returns:
+            Validated EvalConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "app_id": data.get("app_id"),
+            "model": data.get("model"),
+            "seeds": data.get("seeds", []),
+            "task_app_url": data.get("task_app_url"),
+            "split": data.get("split", "train"),
+            "max_turns": data.get("max_turns"),
+            "max_llm_calls": data.get("max_llm_calls", 10),
+            "concurrency": data.get("concurrency", 1),
+            "env_name": data.get("env_name"),
+            "policy_name": data.get("policy_name"),
+            "trace_format": data.get("trace_format", "compact"),
+            "return_trace": data.get("return_trace", False),
+            "ops": data.get("ops"),
+            "env_config": data.get("env_config", {}),
+            "policy_config": data.get("policy_config", {}),
+            "metadata": data.get("metadata", {}),
+            "metadata_sql": data.get("metadata_sql"),
+        }
+        
+        return cls(**config_dict)
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    """Configuration for 'synth-ai filter' command.
+    
+    Validates and provides defaults for filtering traces into SFT datasets.
+    """
+    
+    # Required: Database path or URL
+    db: str
+    
+    # Required: Output JSONL path
+    output: str
+    
+    # Optional: Filter by data splits
+    splits: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by task IDs
+    task_ids: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by models
+    models: list[str] = field(default_factory=list)
+    
+    # Optional: Minimum official score threshold
+    min_official_score: float | None = None
+    
+    # Optional: Maximum official score threshold
+    max_official_score: float | None = None
+    
+    # Optional: Minimum judge scores (judge_name -> min_score)
+    min_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Maximum judge scores (judge_name -> max_score)
+    max_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Limit number of examples
+    limit: int | None = None
+    
+    # Optional: Offset for pagination
+    offset: int | None = None
+    
+    # Optional: Whether to shuffle results
+    shuffle: bool = False
+    
+    # Optional: Random seed for shuffling
+    shuffle_seed: int | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.db:
+            raise ValueError("db (database path or URL) is required")
+        
+        if not self.output:
+            raise ValueError("output (JSONL file path) is required")
+        
+        # Validate output has .jsonl extension
+        output_path = Path(self.output)
+        if output_path.suffix.lower() not in (".jsonl", ".json"):
+            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
+        
+        # Validate score thresholds
+        if (
+            self.min_official_score is not None
+            and self.max_official_score is not None
+            and self.min_official_score > self.max_official_score
+        ):
+            raise ValueError("min_official_score cannot be greater than max_official_score")
+        
+        # Validate limit/offset
+        if self.limit is not None and self.limit < 1:
+            raise ValueError("limit must be >= 1")
+        
+        if self.offset is not None and self.offset < 0:
+            raise ValueError("offset must be >= 0")
+        
+        # Validate shuffle seed requires shuffle
+        if self.shuffle_seed is not None and not self.shuffle:
+            raise ValueError("shuffle_seed requires shuffle=true")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
+        """Create FilterConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with filter configuration
+            
+        Returns:
+            Validated FilterConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "db": data.get("db"),
+            "output": data.get("output"),
+            "splits": data.get("splits", []),
+            "task_ids": data.get("task_ids", []),
+            "models": data.get("models", []),
+            "min_official_score": data.get("min_official_score"),
+            "max_official_score": data.get("max_official_score"),
+            "min_judge_scores": data.get("min_judge_scores", {}),
+            "max_judge_scores": data.get("max_judge_scores", {}),
+            "limit": data.get("limit"),
+            "offset": data.get("offset"),
+            "shuffle": data.get("shuffle", False),
+            "shuffle_seed": data.get("shuffle_seed"),
+        }
+        
+        return cls(**config_dict)
+    
+    def get_db_url(self) -> str:
+        """Convert db path to proper SQLite URL if needed.
+        
+        Returns:
+            Database URL suitable for SQLAlchemy/aiosqlite
+        """
+        db_value = self.db.strip()
+        if "://" in db_value:
+            return db_value
+        else:
+            db_path = Path(db_value).expanduser().resolve()
+            # Ensure parent directory exists
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            return f"sqlite+aiosqlite:///{db_path}"
+    
+    def get_output_path(self) -> Path:
+        """Get resolved output path with parent directory created.
+        
+        Returns:
+            Resolved Path object with parent directory created
+        """
+        output_path = Path(self.output).expanduser().resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+

synth_ai/task/contracts.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from enum import Enum
 from typing import Any, Literal
 
 from pydantic import BaseModel, ConfigDict, Field
 
 
+class RolloutMode(str, Enum):
+    """Mode controls how rollout infrastructure processes inference URLs."""
+    RL = "rl"
+    EVAL = "eval"
+
+
 @dataclass(frozen=True)
 class TaskAppEndpoints:
     """Required Task App endpoints used by RL trainers and clients.
@@ -43,7 +50,7 @@ class RolloutRecordConfig(BaseModel):
     logprobs: bool = False
     value: bool = False
     return_trace: bool = False
-    trace_format: Literal["compact", "full"] = "compact"
+    trace_format: Literal["compact", "full", "structured"] = "compact"
 
 
 class RolloutSafetyConfig(BaseModel):
@@ -61,6 +68,7 @@ class RolloutRequest(BaseModel):
     safety: RolloutSafetyConfig = RolloutSafetyConfig()
     training_session_id: str | None = None
     synth_base_url: str | None = None
+    mode: RolloutMode  # Required: explicit RL vs EVAL mode
 
 
 class RolloutStep(BaseModel):
@@ -110,7 +118,7 @@ class RolloutTrajectory(BaseModel):
     
     # Required for trace correlation with inference mesh (optional initially for backward compat)
     # See: monorepo/INFERENCE_URL_REQUIREMENT_PLAN.md and trace_creation_and_judgement.txt
-    inference_url: str | None = None
+    inference_url: str
     
     decision_samples: list[dict[str, Any]] | None = None
 
@@ -143,10 +151,15 @@ class RolloutResponse(BaseModel):
     aborted: bool = False
     ops_executed: int = 0
     
+    # OPTIONAL: correlation ID for linking rollout to inference traces
+    # If not provided, trainer will infer it from trajectory.inference_url ?cid=... parameter
+    trace_correlation_id: str | None = None
+    
     # PREFERRED: v3 trace format (SessionTrace). This is the single source of truth
     # for rollout data and should be used by all new code. Contains richer data than
     # trajectories including token IDs, logprobs, timing, and multimodal content.
     trace: dict[str, Any] | None = None
+    pipeline_metadata: dict[str, Any] = Field(default_factory=dict)
 
 
 class _ExtraAllowModel(BaseModel):

synth_ai/task/rubrics/__init__.py

@@ -9,10 +9,9 @@ This module provides:
 """
 
 # Core models (flexible validation)
-from .models import Criterion, Rubric
-
 # Loading and blending
 from .loaders import blend_rubrics, load_rubric
+from .models import Criterion, Rubric
 
 # Scoring
 from .scoring import score_events_against_rubric, score_outcome_against_rubric
@@ -51,3 +50,6 @@ __all__ = [
 RubricCriterion = StrictCriterion
 RubricSpec = StrictRubric
 
+
+
+

synth_ai/task/rubrics/loaders.py

@@ -60,15 +60,39 @@ def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
         
     Returns:
         Parsed Rubric instance or None if source is None
+        
+    Raises:
+        ValueError: If the rubric format is incorrect (e.g., backend judge format)
+        ValidationError: If the rubric fails schema validation
     """
     if source is None:
         return None
     if isinstance(source, Rubric):
         return source
+    
+    # Load and parse the data
     if isinstance(source, dict):
-        return Rubric.model_validate(source)
-    text, suffix = _load_text(str(source))
-    data = _parse_structured(text, suffix)
+        data = source
+    else:
+        text, suffix = _load_text(str(source))
+        data = _parse_structured(text, suffix)
+    
+    # Check if this looks like a backend judge rubric (wrong format)
+    if (
+        isinstance(data, dict)
+        and "event" in data
+        and "outcome" in data
+        and "version" not in data
+        and "goal_text" not in data
+        and "criteria" not in data
+    ):
+        source_hint = f" ({source})" if isinstance(source, str) else ""
+        raise ValueError(
+            f"Rubric appears to be in backend judge format (has 'event'/'outcome' keys){source_hint}. "
+            f"Task apps require rubrics with 'version', 'goal_text', and 'criteria' fields. "
+            f"Backend judge rubrics should be named '*_backend_judge.json' and loaded by judge functions."
+        )
+    
     return Rubric.model_validate(data)
 
 
@@ -130,4 +154,3 @@ def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None
         criteria=merged,
         aggregation=aggregation,
     )
-

synth_ai/task/rubrics/scoring.py

@@ -111,3 +111,6 @@ def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None)
                     values[str(key)] = score
     return _score(rubric.criteria, values, rubric.aggregation)
 
+
+
+

synth_ai/task/rubrics.py

@@ -0,0 +1,219 @@
+"""Rubric schema, loading, and scoring helpers for Task Apps."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Criterion(BaseModel):
+    id: str
+    description: str
+    weight: float = 1.0
+    required: bool = False
+
+    @field_validator("weight")
+    @classmethod
+    def _validate_weight(cls, value: float) -> float:
+        if value <= 0:
+            raise ValueError("criterion weight must be positive")
+        return value
+
+
+class Rubric(BaseModel):
+    version: str
+    goal_text: str | None = None
+    criteria: list[Criterion] = Field(default_factory=list)
+    aggregation: str = "weighted_sum"
+
+    @field_validator("aggregation")
+    @classmethod
+    def _validate_aggregation(cls, value: str) -> str:
+        allowed = {"sum", "weighted_sum", "custom", "inherit"}
+        if value not in allowed:
+            raise ValueError(f"aggregation must be one of {sorted(allowed)}")
+        return value
+
+    @field_validator("criteria")
+    @classmethod
+    def _validate_criteria(cls, criteria: list[Criterion]) -> list[Criterion]:
+        seen = set()
+        for criterion in criteria:
+            if criterion.id in seen:
+                raise ValueError(f"duplicate criterion id: {criterion.id}")
+            seen.add(criterion.id)
+        return criteria
+
+
+def _load_text(source: str) -> tuple[str, str | None]:
+    path = Path(source)
+    if path.exists():
+        return path.read_text(encoding="utf-8"), path.suffix.lower()
+    return source, None
+
+
+def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
+    text = text.strip()
+    if not text:
+        raise ValueError("Rubric source is empty")
+    if suffix in (".yaml", ".yml"):
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric YAML must produce a mapping") from None
+        return data
+    if text.startswith("{"):
+        return json.loads(text)
+    if text.startswith("http://") or text.startswith("https://"):
+        import requests  # type: ignore
+
+        response = requests.get(text, timeout=15)
+        response.raise_for_status()
+        return _parse_structured(response.text, suffix)
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        try:
+            import yaml  # type: ignore
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise RuntimeError("PyYAML is required to load rubric text") from exc
+        data = yaml.safe_load(text)
+        if not isinstance(data, dict):
+            raise ValueError("Rubric text must decode to a mapping") from None
+        return data
+
+
+def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
+    if source is None:
+        return None
+    if isinstance(source, Rubric):
+        return source
+    if isinstance(source, dict):
+        return Rubric.model_validate(source)
+    text, suffix = _load_text(str(source))
+    data = _parse_structured(text, suffix)
+    return Rubric.model_validate(data)
+
+
+def _merge_weights(base: Criterion, override: Criterion) -> float:
+    if override.weight != 1.0 and base.weight != 1.0:
+        return base.weight * override.weight
+    if override.weight != 1.0:
+        return override.weight
+    return base.weight
+
+
+def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None:
+    if override is None and base is None:
+        return None
+    if base is None:
+        return override
+    if override is None:
+        return base
+
+    base_map = {criterion.id: criterion for criterion in base.criteria}
+    merged: list[Criterion] = []
+
+    for ov in override.criteria:
+        if ov.id in base_map:
+            existing = base_map.pop(ov.id)
+            merged.append(
+                Criterion(
+                    id=ov.id,
+                    description=ov.description or existing.description,
+                    weight=_merge_weights(existing, ov),
+                    required=ov.required if ov.required is not None else existing.required,
+                )
+            )
+        else:
+            merged.append(ov)
+
+    merged.extend(base_map.values())
+
+    aggregation = override.aggregation
+    if aggregation == "inherit":
+        aggregation = base.aggregation
+
+    return Rubric(
+        version=override.version or base.version,
+        goal_text=override.goal_text or base.goal_text,
+        criteria=merged,
+        aggregation=aggregation,
+    )
+
+
+def _as_float(value: Any) -> float | None:
+    try:
+        return float(value)
+    except Exception:
+        return None
+
+
+def _score(
+    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
+) -> dict[str, Any]:
+    if aggregation == "inherit":
+        aggregation = "weighted_sum"
+    per_criterion: dict[str, dict[str, Any]] = {}
+    total = 0.0
+    total_weight = 0.0
+    for criterion in criteria:
+        score = values.get(criterion.id, 0.0)
+        per_criterion[criterion.id] = {
+            "score": score,
+            "weight": criterion.weight,
+            "required": criterion.required,
+        }
+        if aggregation == "sum":
+            total += score
+        elif aggregation == "weighted_sum":
+            total += score * criterion.weight
+            total_weight += criterion.weight
+    if aggregation == "weighted_sum" and total_weight > 0:
+        total = total / total_weight
+    if aggregation == "custom":
+        total = None  # type: ignore[assignment]
+    return {
+        "aggregation": aggregation,
+        "score": total,
+        "per_criterion": per_criterion,
+    }
+
+
+def score_events_against_rubric(
+    events: list[dict[str, Any]], rubric: Rubric | None
+) -> dict[str, Any]:
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    for event in events or []:
+        if not isinstance(event, dict):
+            continue
+        cid = event.get("criterion_id") or event.get("id") or event.get("criterion")
+        score = _as_float(event.get("score"))
+        if cid and score is not None:
+            values[str(cid)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)
+
+
+def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
+    if rubric is None:
+        return {"aggregation": "none", "score": None, "per_criterion": {}}
+    values: dict[str, float] = {}
+    if isinstance(outcome, dict):
+        candidates = (
+            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
+        )
+        if isinstance(candidates, dict):
+            for key, value in candidates.items():
+                score = _as_float(value)
+                if score is not None:
+                    values[str(key)] = score
+    return _score(rubric.criteria, values, rubric.aggregation)