synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/cli/commands/train/judge_validation.py

@@ -1,305 +0,0 @@
-"""
-Validation logic for judge/rubric configuration from TOML.
-
-This module validates and normalizes judge/rubric config, removing all dead fields
-and ensuring only the fields actually used by the backend are present.
-"""
-
-from __future__ import annotations
-
-import warnings
-from collections.abc import MutableMapping
-from typing import Any, Optional, Tuple
-
-from pydantic import ValidationError
-
-from .errors import InvalidJudgeConfigError, InvalidRubricConfigError
-from .judge_schemas import JudgeConfig, JudgeOptionsConfig, RubricConfig, RubricWeightsConfig
-
-__all__ = [
-    "validate_judge_config",
-    "validate_rubric_config",
-    "extract_and_validate_judge_rubric",
-]
-
-# Dead fields that should trigger deprecation warnings
-DEPRECATED_RUBRIC_FIELDS = {
-    "model",
-    "api_base",
-    "api_key_env",
-    "event",
-    "outcome",
-}
-
-DEPRECATED_JUDGE_FIELDS = {
-    "type",
-    "timeout_s",  # Moved to judge.options.timeout_s
-}
-
-DEPRECATED_JUDGE_OPTIONS_FIELDS = {
-    "max_concurrency",
-    "tracks",
-}
-
-
-def _warn_deprecated_fields(section: str, fields: set[str], present_fields: set[str]) -> None:
-    """Emit deprecation warnings for dead fields that are present in config."""
-    deprecated_present = fields & present_fields
-    if deprecated_present:
-        field_list = ", ".join(sorted(deprecated_present))
-        warnings.warn(
-            f"[{section}] contains deprecated fields that are no longer used: {field_list}. "
-            f"These fields will be ignored and should be removed from your config. "
-            f"See judge/rubric cleanup guide for details.",
-            DeprecationWarning,
-            stacklevel=3,
-        )
-
-
-def validate_rubric_config(config: MutableMapping[str, Any]) -> RubricConfig:
-    """
-    Validate and normalize rubric configuration from TOML.
-    
-    Args:
-        config: Raw [rubric] section from TOML
-        
-    Returns:
-        Validated RubricConfig instance
-        
-    Raises:
-        InvalidRubricConfigError: If validation fails
-    """
-    if not config:
-        # Default: rubric disabled
-        return RubricConfig(enabled=False)
-    
-    config_dict = dict(config)
-    
-    # Warn about deprecated fields
-    _warn_deprecated_fields("rubric", DEPRECATED_RUBRIC_FIELDS, set(config_dict.keys()))
-    
-    # Warn about deprecated subsections
-    if "event" in config_dict:
-        warnings.warn(
-            "[rubric.event] section is deprecated and no longer used. "
-            "Criteria are now fetched dynamically from TaskInfo or specified in "
-            "[judge.options.rubric_overrides]. This section will be ignored.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-    
-    if "outcome" in config_dict:
-        warnings.warn(
-            "[rubric.outcome] section is deprecated and no longer used. "
-            "Criteria are now fetched dynamically from TaskInfo or specified in "
-            "[judge.options.rubric_overrides]. This section will be ignored.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-    
-    # Extract only valid fields
-    enabled = config_dict.get("enabled", False)
-    weights_dict = config_dict.get("weights", {})
-    
-    # Validate using Pydantic
-    try:
-        if not isinstance(weights_dict, dict):
-            raise ValueError("[rubric.weights] must be a dictionary")
-        
-        weights = RubricWeightsConfig(**weights_dict)
-        return RubricConfig(enabled=enabled, weights=weights)
-    
-    except ValidationError as exc:
-        errors = []
-        for error in exc.errors():
-            loc = ".".join(str(x) for x in error["loc"])
-            msg = error["msg"]
-            errors.append(f"  • rubric.{loc}: {msg}")
-        raise InvalidRubricConfigError(
-            detail="Rubric validation failed:\n" + "\n".join(errors)
-        ) from exc
-    except Exception as exc:
-        raise InvalidRubricConfigError(
-            detail=f"Rubric validation failed: {exc}"
-        ) from exc
-
-
-def validate_judge_config(config: MutableMapping[str, Any]) -> Optional[JudgeConfig]:
-    """
-    Validate and normalize judge configuration from TOML.
-    
-    Args:
-        config: Raw [judge] section from TOML
-        
-    Returns:
-        Validated JudgeConfig instance, or None if not present
-        
-    Raises:
-        InvalidJudgeConfigError: If validation fails
-    """
-    if not config:
-        return None
-    
-    config_dict = dict(config)
-    
-    # Warn about deprecated top-level fields
-    _warn_deprecated_fields("judge", DEPRECATED_JUDGE_FIELDS, set(config_dict.keys()))
-    
-    # Extract judge.options (required)
-    options_dict = config_dict.get("options")
-    if not options_dict:
-        raise InvalidJudgeConfigError(
-            detail="[judge.options] section is required when [judge] is present"
-        )
-    
-    if not isinstance(options_dict, dict):
-        raise InvalidJudgeConfigError(
-            detail="[judge.options] must be a dictionary"
-        )
-    
-    # Warn about deprecated options fields
-    _warn_deprecated_fields(
-        "judge.options",
-        DEPRECATED_JUDGE_OPTIONS_FIELDS,
-        set(options_dict.keys()),
-    )
-    
-    # Remove deprecated fields from options
-    options_dict = {
-        k: v for k, v in options_dict.items()
-        if k not in DEPRECATED_JUDGE_OPTIONS_FIELDS
-    }
-    
-    # Migrate judge.timeout_s to judge.options.timeout_s if present
-    if "timeout_s" in config_dict and "timeout_s" not in options_dict:
-        warnings.warn(
-            "[judge].timeout_s is deprecated. Use [judge.options].timeout_s instead. "
-            "Auto-migrating for now.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        options_dict["timeout_s"] = config_dict["timeout_s"]
-    
-    # Validate using Pydantic
-    try:
-        options = JudgeOptionsConfig(**options_dict)
-        return JudgeConfig(options=options)
-    
-    except ValidationError as exc:
-        errors = []
-        for error in exc.errors():
-            loc = ".".join(str(x) for x in error["loc"])
-            msg = error["msg"]
-            errors.append(f"  • judge.options.{loc}: {msg}")
-        raise InvalidJudgeConfigError(
-            detail="Judge validation failed:\n" + "\n".join(errors)
-        ) from exc
-    except Exception as exc:
-        raise InvalidJudgeConfigError(
-            detail=f"Judge validation failed: {exc}"
-        ) from exc
-
-
-def extract_and_validate_judge_rubric(
-    toml_config: MutableMapping[str, Any]
-) -> Tuple[RubricConfig, Optional[JudgeConfig]]:
-    """
-    Extract and validate judge/rubric config from full TOML config.
-    
-    Args:
-        toml_config: Full TOML configuration dict
-        
-    Returns:
-        Tuple of (validated_rubric, validated_judge_or_none)
-        
-    Raises:
-        InvalidRubricConfigError: If rubric validation fails
-        InvalidJudgeConfigError: If judge validation fails
-    """
-    rubric_dict = toml_config.get("rubric", {})
-    judge_dict = toml_config.get("judge", {})
-    
-    # Validate rubric
-    rubric_config = validate_rubric_config(rubric_dict)
-    
-    # Validate judge (if present)
-    judge_config = validate_judge_config(judge_dict) if judge_dict else None
-    
-    # Cross-validation: If rubric is enabled, judge options should be present
-    if rubric_config.enabled and not judge_config:
-        warnings.warn(
-            "[rubric].enabled=true but [judge] section is missing. "
-            "Rubric-based judging requires judge configuration. "
-            "Rubric scoring will be disabled.",
-            UserWarning,
-            stacklevel=2,
-        )
-        rubric_config.enabled = False
-    
-    # Cross-validation: Warn if weights don't align with enabled judging types
-    if rubric_config.enabled and judge_config:
-        weights = rubric_config.weights
-        options = judge_config.options
-        
-        if weights.event > 0 and not options.event:
-            warnings.warn(
-                "[rubric.weights].event > 0 but [judge.options].event=false. "
-                "Event-level judge scores will be 0 (no event judging enabled).",
-                UserWarning,
-                stacklevel=2,
-            )
-        
-        if weights.outcome > 0 and not options.outcome:
-            warnings.warn(
-                "[rubric.weights].outcome > 0 but [judge.options].outcome=false. "
-                "Outcome judge score will be 0 (no outcome judging enabled).",
-                UserWarning,
-                stacklevel=2,
-            )
-    
-    return rubric_config, judge_config
-
-
-# Helper to check if config has any deprecated fields (for testing/migration)
-
-def check_for_deprecated_fields(toml_config: MutableMapping[str, Any]) -> dict[str, list[str]]:
-    """
-    Check TOML config for deprecated fields without validation.
-    
-    Returns dict of {section: [deprecated_field_names]} for reporting.
-    """
-    deprecated: dict[str, list[str]] = {}
-    
-    rubric_dict = toml_config.get("rubric", {})
-    if rubric_dict:
-        found = [
-            field for field in DEPRECATED_RUBRIC_FIELDS
-            if field in rubric_dict
-        ]
-        if "event" in rubric_dict:
-            found.append("event (entire section)")
-        if "outcome" in rubric_dict:
-            found.append("outcome (entire section)")
-        if found:
-            deprecated["rubric"] = found
-    
-    judge_dict = toml_config.get("judge", {})
-    if judge_dict:
-        found = [
-            field for field in DEPRECATED_JUDGE_FIELDS
-            if field in judge_dict
-        ]
-        if found:
-            deprecated["judge"] = found
-        
-        options_dict = judge_dict.get("options", {})
-        if options_dict:
-            options_found = [
-                field for field in DEPRECATED_JUDGE_OPTIONS_FIELDS
-                if field in options_dict
-            ]
-            if options_found:
-                deprecated["judge.options"] = options_found
-    
-    return deprecated
-

synth_ai/cli/usage.py

@@ -1,159 +0,0 @@
-"""CLI command for displaying org usage and limits.
-
-Usage:
-    synth-ai usage          # Show usage summary
-    synth-ai usage --json   # Output as JSON
-"""
-
-from __future__ import annotations
-
-import json
-import sys
-from dataclasses import asdict
-
-import click
-
-
-def _format_number(n: int | float) -> str:
-    """Format a number for display (e.g., 1000 -> 1K)."""
-    if n >= 1_000_000:
-        return f"{n / 1_000_000:.1f}M"
-    if n >= 1_000:
-        return f"{n / 1_000:.1f}K"
-    if isinstance(n, float):
-        return f"{n:.1f}"
-    return str(n)
-
-
-def _progress_bar(percent: float, width: int = 20) -> str:
-    """Create a text-based progress bar."""
-    filled = int(width * min(percent, 100) / 100)
-    bar = "=" * filled + "-" * (width - filled)
-    return f"[{bar}]"
-
-
-def _format_metric(name: str, metric, indent: str = "  ") -> str:
-    """Format a single usage metric for display."""
-    percent = metric.percent_used
-    bar = _progress_bar(percent)
-    used_str = _format_number(metric.used)
-    limit_str = _format_number(metric.limit)
-    return f"{indent}{name}: {bar} {used_str} / {limit_str} ({percent:.1f}%)"
-
-
-@click.command()
-@click.option(
-    "--json",
-    "as_json",
-    is_flag=True,
-    default=False,
-    help="Output as JSON",
-)
-def usage_cmd(as_json: bool) -> None:
-    """Display org usage and rate limits.
-
-    Shows current usage across all Synth APIs with progress bars
-    indicating how much of each limit has been consumed.
-    """
-    try:
-        from synth_ai.sdk.usage import UsageClient
-    except ImportError as e:
-        click.echo(f"Error importing usage client: {e}", err=True)
-        sys.exit(1)
-
-    try:
-        client = UsageClient()
-        usage = client.get()
-    except Exception as e:
-        click.echo(f"Error fetching usage: {e}", err=True)
-        sys.exit(1)
-
-    if as_json:
-        # Convert to dict and output as JSON
-        output = {
-            "org_id": usage.org_id,
-            "tier": usage.tier,
-            "period": {
-                "daily_start": usage.period.daily_start.isoformat(),
-                "monthly_start": usage.period.monthly_start.isoformat(),
-            },
-            "apis": {
-                "inference": {
-                    "requests_per_min": asdict(usage.apis.inference.requests_per_min),
-                    "tokens_per_day": asdict(usage.apis.inference.tokens_per_day),
-                    "spend_cents_per_month": asdict(usage.apis.inference.spend_cents_per_month),
-                },
-                "judges": {
-                    "evaluations_per_day": asdict(usage.apis.judges.evaluations_per_day),
-                },
-                "prompt_opt": {
-                    "jobs_per_day": asdict(usage.apis.prompt_opt.jobs_per_day),
-                    "rollouts_per_day": asdict(usage.apis.prompt_opt.rollouts_per_day),
-                    "spend_cents_per_day": asdict(usage.apis.prompt_opt.spend_cents_per_day),
-                },
-                "rl": {
-                    "jobs_per_month": asdict(usage.apis.rl.jobs_per_month),
-                    "gpu_hours_per_month": asdict(usage.apis.rl.gpu_hours_per_month),
-                },
-                "sft": {
-                    "jobs_per_month": asdict(usage.apis.sft.jobs_per_month),
-                    "gpu_hours_per_month": asdict(usage.apis.sft.gpu_hours_per_month),
-                },
-                "research": {
-                    "jobs_per_month": asdict(usage.apis.research.jobs_per_month),
-                    "agent_spend_cents_per_month": asdict(usage.apis.research.agent_spend_cents_per_month),
-                },
-            },
-            "totals": {
-                "spend_cents_per_month": asdict(usage.totals.spend_cents_per_month),
-            },
-        }
-        click.echo(json.dumps(output, indent=2))
-        return
-
-    # Pretty print format
-    click.echo("")
-    click.echo(f"Org: {usage.org_id} ({usage.tier} tier)")
-    click.echo("")
-
-    # Inference
-    click.echo("Inference")
-    click.echo(_format_metric("requests/min", usage.apis.inference.requests_per_min))
-    click.echo(_format_metric("tokens/day", usage.apis.inference.tokens_per_day))
-    click.echo(_format_metric("spend/month", usage.apis.inference.spend_cents_per_month))
-    click.echo("")
-
-    # Judges
-    click.echo("Judges")
-    click.echo(_format_metric("evaluations/day", usage.apis.judges.evaluations_per_day))
-    click.echo("")
-
-    # Prompt Optimization
-    click.echo("Prompt Optimization")
-    click.echo(_format_metric("jobs/day", usage.apis.prompt_opt.jobs_per_day))
-    click.echo(_format_metric("rollouts/day", usage.apis.prompt_opt.rollouts_per_day))
-    click.echo(_format_metric("spend/day", usage.apis.prompt_opt.spend_cents_per_day))
-    click.echo("")
-
-    # RL Training
-    click.echo("RL Training")
-    click.echo(_format_metric("jobs/month", usage.apis.rl.jobs_per_month))
-    click.echo(_format_metric("gpu_hours/month", usage.apis.rl.gpu_hours_per_month))
-    click.echo("")
-
-    # SFT Training
-    click.echo("SFT Training")
-    click.echo(_format_metric("jobs/month", usage.apis.sft.jobs_per_month))
-    click.echo(_format_metric("gpu_hours/month", usage.apis.sft.gpu_hours_per_month))
-    click.echo("")
-
-    # Research Agents
-    click.echo("Research Agents")
-    click.echo(_format_metric("jobs/month", usage.apis.research.jobs_per_month))
-    click.echo(_format_metric("agent_spend/month", usage.apis.research.agent_spend_cents_per_month))
-    click.echo("")
-
-    # Totals
-    click.echo("Total Spend")
-    click.echo(_format_metric("spend/month", usage.totals.spend_cents_per_month))
-    click.echo("")

synth_ai/data/specs.py

@@ -1,36 +0,0 @@
-"""System specification data types.
-
-This module re-exports spec dataclasses from their original location
-to provide a cleaner import path: `from synth_ai.data.specs import Spec`
-
-Based on Sean Grove's "spec as code" pattern from AI Engineer World's Fair.
-Specs are the source of truth that encode intent, policies, and rules.
-"""
-
-from __future__ import annotations
-
-from synth_ai.sdk.specs.dataclasses import (
-    Constraints,
-    Example,
-    GlossaryItem,
-    Interfaces,
-    Metadata,
-    Principle,
-    Rule,
-    Spec,
-    TestCase,
-)
-
-__all__ = [
-    "Spec",
-    "Metadata",
-    "Principle",
-    "Rule",
-    "Constraints",
-    "Example",
-    "TestCase",
-    "Interfaces",
-    "GlossaryItem",
-]
-
-