synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

examples/agora_ex/task_app/reward_fn_grpo-human.py

@@ -1,129 +0,0 @@
-import logging
-import requests
-from typing import Any, Dict, Union
-
-# ---------------------------------------------------------------------------
-# Run configuration defaults (override via kwargs when invoking reward_fn)
-# ---------------------------------------------------------------------------
-RUN_TYPE: str = "rl_training_human"
-RUN_VERSION: float = 3.5
-MODEL_NAME: str = "Qwen3-30B-A3B-Instruct"
-EXPERIMENT_NAME: str = "qwen3_30b_human"
-USER_PROMPT_VERSION: str = "5.0"
-SYSTEM_PROMPT_VERSION: str = "4.0"
-
-logger = logging.getLogger(__name__)
-
-
-def _coerce_step_value(value: Any) -> Union[int, None]:
-    try:
-        if value is None:
-            return None
-        return int(value)
-    except (TypeError, ValueError):
-        return None
-
-
-def _build_metadata(kwargs: Dict[str, Any]) -> Dict[str, Any]:
-    """Compose the metadata payload sent with the evaluation request."""
-    base_metadata: Dict[str, Any] = {
-        "model": kwargs.get("model", MODEL_NAME),
-        "experiment": kwargs.get("experiment", EXPERIMENT_NAME),
-        "step_number": kwargs.get("step_number"),
-        "user_prompt": kwargs.get("user_prompt", USER_PROMPT_VERSION),
-        "batch_number": kwargs.get("batch_number"),
-        "prompt_index": kwargs.get("prompt_index"),
-        "rollout_group": kwargs.get("rollout_group"),
-        "system_prompt": kwargs.get("system_prompt", SYSTEM_PROMPT_VERSION),
-    }
-
-    step_metadata = kwargs.get("metadata") or {}
-    if isinstance(step_metadata, dict):
-        # Map common harness metadata onto our schema when present
-        step_value = _coerce_step_value(
-            step_metadata.get("step") or step_metadata.get("step_number")
-        )
-        if step_value is not None:
-            base_metadata["step_number"] = step_value
-
-        rollout_value = step_metadata.get("rollout_group")
-        if rollout_value is not None:
-            base_metadata["rollout_group"] = rollout_value
-
-        extras = step_metadata.get("extras")
-        if extras:
-            base_metadata["extras"] = extras
-
-        # Preserve any additional custom metadata fields
-        for key, value in step_metadata.items():
-            if key in {"step", "step_number", "rollout_group", "extras"}:
-                continue
-            if key not in base_metadata or base_metadata.get(key) is None:
-                base_metadata[key] = value
-
-    # Strip keys that remain None so the JSON is clean
-    return {key: value for key, value in base_metadata.items() if value is not None}
-
-
-def reward_fn(
-    completion: str,
-    **kwargs,
-) -> float:
-    """Evaluate the model response and return a reward score (0.0-1.0)."""
-    run_type = kwargs.get("run_type", RUN_TYPE)
-    run_version = kwargs.get("run_version", RUN_VERSION)
-    metadata = _build_metadata(kwargs)
-
-    payload = {
-        "code": completion,
-        "run_type": run_type,
-        "run_version": run_version,
-        "metadata": metadata,
-    }
-
-    try:
-        response = requests.post(
-            "https://eames-judge-api-769874896543.us-central1.run.app/evaluations-human",
-            json=payload,
-            timeout=1800,  # 30 minute timeout for screenshot generation
-        )
-        response.raise_for_status()
-        result = response.json()
-
-        logger.info("Evaluation complete:")
-        logger.info("  Score: %s", result.get("score", 0.0))
-        logger.info("  Feedback: %s", result.get("explanation", "N/A"))
-        logger.info("  Processing Time (ms): %s", result.get("processing_time_ms", "N/A"))
-        logger.info("  Worker ID: %s", result.get("worker_id", "N/A"))
-        logger.info("  Success: %s", result.get("success", False))
-        if metadata:
-            logger.info("  Metadata sent: %s", metadata)
-
-        screenshot_urls = result.get("screenshot_urls", {}) or {}
-        if screenshot_urls:
-            logger.info("  Screenshot URLs:")
-            for key, url in screenshot_urls.items():
-                logger.info("    %s: %s", key.capitalize(), url)
-
-        score = result.get("score", 0.0)
-        if not isinstance(score, (int, float)):
-            logger.warning("Invalid score type: %s. Defaulting to 0.0", type(score))
-            return 0.0
-
-        return max(0.0, min(1.0, float(score)))
-
-    except requests.exceptions.Timeout:
-        logger.error("Request to evaluation server timed out")
-        return 0.0
-
-    except requests.exceptions.RequestException as exc:
-        logger.error("Request to evaluation server failed: %s", exc)
-        return 0.0
-
-    except (KeyError, ValueError, TypeError) as exc:
-        logger.error("Error parsing evaluation server response: %s", exc)
-        return 0.0
-
-    except Exception as exc:  # pylint: disable=broad-except
-        logger.error("Unexpected error in reward_fn: %s", exc)
-        return 0.0

examples/agora_ex/task_app/system_prompt_CURRENT.md

@@ -1,63 +0,0 @@
-You are a powerful agentic AI coding assistant called Eames working with a Next.js + Shadcn/UI TypeScript project to generate exactly one complete landing page file for SaaS/Software products.
-
-## OUTPUT FORMAT
-
-Return ONLY a single TypeScript React component file.
-- Wrap in one code fence: ```tsx ... ```
-- No explanations, no additional text before or after the fence
-- If you cannot generate valid code, return nothing
-
-## FILE SPECIFICATION
-
-File: app/page.tsx
-Length: Target 800-1200 lines (flexible based on content richness)
-Tech: Next.js 14 App Router, TypeScript, React, Tailwind CSS only
-**IMPORTANT:** DO NOT use images, image imports, or next/image. Use SVG icons, Tailwind patterns, gradients, or CSS shapes instead.
-
-## REQUIRED STRUCTURE (in this exact order)
-
-1. Imports: NONE (do not import next/image or any other libraries)
-
-2. Metadata export:
-   ```tsx
-   export const metadata = {
-     title: "Page Title (max 80 chars)",
-     description: "Page description (max 160 chars)"
-   }
-   ```
-3. Helper components (if needed): Define small inline components AFTER the Page export
-   - Examples: FeatureCard, PricingCard, TestimonialCard, LogoItem
-   - Keep minimal, no deep nesting
-
-## DESIGN GUIDELINES
-Ship something interesting rather than boring, but never ugly.
-Include images and SVGs that are relevant to the category of business.
-
-## TECHNICAL CONSTRAINTS
-
-✓ Server component by default (no "use client" unless interactive state/events needed)
-✓ Tailwind utility classes for ALL styling
-✓ Semantic HTML5: `<main>`, `<section>`, `<header>`, `<footer>`, `<h1>`-`<h6>` hierarchy
-✓ Use inline SVG for icons (simple shapes: circles, squares, arrows, checkmarks, etc.)
-✓ Use Tailwind gradients, borders, and shadows for visual elements
-✓ Use CSS shapes and patterns instead of images
-
-✗ NO images - do not use `<img>`, `<Image>`, or any image imports
-✗ NO next/image imports
-✗ No data fetching (fetch, axios, server actions)
-✗ No lorem ipsum - write real, specific copy
-
-## VALIDATION
-
-Your output must:
-1. Be a single valid .tsx file with NO imports whatsoever
-2. Include `export const metadata`
-3. Include `export default function Page()`
-4. Include all 8 required sections in order: Navbar, Hero, Logos, Features, Testimonials, Pricing, Final CTA, Footer
-5. Use only Tailwind for styling
-6. Be deployable in Next.js 14 App Router without errors
-7. Use proper TypeScript syntax
-8. Follow the specific product category requested in the user prompt
-9. **NO images** - use SVG icons and Tailwind styling only
-
-Given the user's prompt describing the website theme/product, generate the code immediately.

synth_ai/rubrics/__init__.py

@@ -1,22 +0,0 @@
-"""
-Rubric utilities.
-
-Exposes helpers for validating rubric specifications that are used across
-Crafter-style judge configurations.
-"""
-
-from .validators import (
-    RubricCriterion,
-    RubricSpec,
-    ValidationError,
-    validate_rubric_dict,
-    validate_rubric_file,
-)
-
-__all__ = [
-    "RubricCriterion",
-    "RubricSpec",
-    "ValidationError",
-    "validate_rubric_dict",
-    "validate_rubric_file",
-]

synth_ai/task/rubrics.py

@@ -1,219 +0,0 @@
-"""Rubric schema, loading, and scoring helpers for Task Apps."""
-
-from __future__ import annotations
-
-import json
-from collections.abc import Iterable
-from pathlib import Path
-from typing import Any
-
-from pydantic import BaseModel, Field, field_validator
-
-
-class Criterion(BaseModel):
-    id: str
-    description: str
-    weight: float = 1.0
-    required: bool = False
-
-    @field_validator("weight")
-    @classmethod
-    def _validate_weight(cls, value: float) -> float:
-        if value <= 0:
-            raise ValueError("criterion weight must be positive")
-        return value
-
-
-class Rubric(BaseModel):
-    version: str
-    goal_text: str | None = None
-    criteria: list[Criterion] = Field(default_factory=list)
-    aggregation: str = "weighted_sum"
-
-    @field_validator("aggregation")
-    @classmethod
-    def _validate_aggregation(cls, value: str) -> str:
-        allowed = {"sum", "weighted_sum", "custom", "inherit"}
-        if value not in allowed:
-            raise ValueError(f"aggregation must be one of {sorted(allowed)}")
-        return value
-
-    @field_validator("criteria")
-    @classmethod
-    def _validate_criteria(cls, criteria: list[Criterion]) -> list[Criterion]:
-        seen = set()
-        for criterion in criteria:
-            if criterion.id in seen:
-                raise ValueError(f"duplicate criterion id: {criterion.id}")
-            seen.add(criterion.id)
-        return criteria
-
-
-def _load_text(source: str) -> tuple[str, str | None]:
-    path = Path(source)
-    if path.exists():
-        return path.read_text(encoding="utf-8"), path.suffix.lower()
-    return source, None
-
-
-def _parse_structured(text: str, suffix: str | None) -> dict[str, Any]:
-    text = text.strip()
-    if not text:
-        raise ValueError("Rubric source is empty")
-    if suffix in (".yaml", ".yml"):
-        try:
-            import yaml  # type: ignore
-        except Exception as exc:  # pragma: no cover - optional dependency
-            raise RuntimeError("PyYAML is required to load YAML rubrics") from exc
-        data = yaml.safe_load(text)
-        if not isinstance(data, dict):
-            raise ValueError("Rubric YAML must produce a mapping") from None
-        return data
-    if text.startswith("{"):
-        return json.loads(text)
-    if text.startswith("http://") or text.startswith("https://"):
-        import requests  # type: ignore
-
-        response = requests.get(text, timeout=15)
-        response.raise_for_status()
-        return _parse_structured(response.text, suffix)
-    try:
-        return json.loads(text)
-    except json.JSONDecodeError:
-        try:
-            import yaml  # type: ignore
-        except Exception as exc:  # pragma: no cover - optional dependency
-            raise RuntimeError("PyYAML is required to load rubric text") from exc
-        data = yaml.safe_load(text)
-        if not isinstance(data, dict):
-            raise ValueError("Rubric text must decode to a mapping") from None
-        return data
-
-
-def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
-    if source is None:
-        return None
-    if isinstance(source, Rubric):
-        return source
-    if isinstance(source, dict):
-        return Rubric.model_validate(source)
-    text, suffix = _load_text(str(source))
-    data = _parse_structured(text, suffix)
-    return Rubric.model_validate(data)
-
-
-def _merge_weights(base: Criterion, override: Criterion) -> float:
-    if override.weight != 1.0 and base.weight != 1.0:
-        return base.weight * override.weight
-    if override.weight != 1.0:
-        return override.weight
-    return base.weight
-
-
-def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None:
-    if override is None and base is None:
-        return None
-    if base is None:
-        return override
-    if override is None:
-        return base
-
-    base_map = {criterion.id: criterion for criterion in base.criteria}
-    merged: list[Criterion] = []
-
-    for ov in override.criteria:
-        if ov.id in base_map:
-            existing = base_map.pop(ov.id)
-            merged.append(
-                Criterion(
-                    id=ov.id,
-                    description=ov.description or existing.description,
-                    weight=_merge_weights(existing, ov),
-                    required=ov.required if ov.required is not None else existing.required,
-                )
-            )
-        else:
-            merged.append(ov)
-
-    merged.extend(base_map.values())
-
-    aggregation = override.aggregation
-    if aggregation == "inherit":
-        aggregation = base.aggregation
-
-    return Rubric(
-        version=override.version or base.version,
-        goal_text=override.goal_text or base.goal_text,
-        criteria=merged,
-        aggregation=aggregation,
-    )
-
-
-def _as_float(value: Any) -> float | None:
-    try:
-        return float(value)
-    except Exception:
-        return None
-
-
-def _score(
-    criteria: Iterable[Criterion], values: dict[str, float], aggregation: str
-) -> dict[str, Any]:
-    if aggregation == "inherit":
-        aggregation = "weighted_sum"
-    per_criterion: dict[str, dict[str, Any]] = {}
-    total = 0.0
-    total_weight = 0.0
-    for criterion in criteria:
-        score = values.get(criterion.id, 0.0)
-        per_criterion[criterion.id] = {
-            "score": score,
-            "weight": criterion.weight,
-            "required": criterion.required,
-        }
-        if aggregation == "sum":
-            total += score
-        elif aggregation == "weighted_sum":
-            total += score * criterion.weight
-            total_weight += criterion.weight
-    if aggregation == "weighted_sum" and total_weight > 0:
-        total = total / total_weight
-    if aggregation == "custom":
-        total = None  # type: ignore[assignment]
-    return {
-        "aggregation": aggregation,
-        "score": total,
-        "per_criterion": per_criterion,
-    }
-
-
-def score_events_against_rubric(
-    events: list[dict[str, Any]], rubric: Rubric | None
-) -> dict[str, Any]:
-    if rubric is None:
-        return {"aggregation": "none", "score": None, "per_criterion": {}}
-    values: dict[str, float] = {}
-    for event in events or []:
-        if not isinstance(event, dict):
-            continue
-        cid = event.get("criterion_id") or event.get("id") or event.get("criterion")
-        score = _as_float(event.get("score"))
-        if cid and score is not None:
-            values[str(cid)] = score
-    return _score(rubric.criteria, values, rubric.aggregation)
-
-
-def score_outcome_against_rubric(outcome: dict[str, Any], rubric: Rubric | None) -> dict[str, Any]:
-    if rubric is None:
-        return {"aggregation": "none", "score": None, "per_criterion": {}}
-    values: dict[str, float] = {}
-    if isinstance(outcome, dict):
-        candidates = (
-            outcome.get("criteria") if isinstance(outcome.get("criteria"), dict) else outcome
-        )
-        if isinstance(candidates, dict):
-            for key, value in candidates.items():
-                score = _as_float(value)
-                if score is not None:
-                    values[str(key)] = score
-    return _score(rubric.criteria, values, rubric.aggregation)