synth-ai 0.2.10__py3-none-any.whl → 0.2.13.dev1__py3-none-any.whl

examples/agora_ex/task_app/reward_fn_grpo-human.py

@@ -0,0 +1,129 @@
+import logging
+import requests
+from typing import Any, Dict, Union
+
+# ---------------------------------------------------------------------------
+# Run configuration defaults (override via kwargs when invoking reward_fn)
+# ---------------------------------------------------------------------------
+RUN_TYPE: str = "rl_training_human"
+RUN_VERSION: float = 3.5
+MODEL_NAME: str = "Qwen3-30B-A3B-Instruct"
+EXPERIMENT_NAME: str = "qwen3_30b_human"
+USER_PROMPT_VERSION: str = "5.0"
+SYSTEM_PROMPT_VERSION: str = "4.0"
+
+logger = logging.getLogger(__name__)
+
+
+def _coerce_step_value(value: Any) -> Union[int, None]:
+    try:
+        if value is None:
+            return None
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _build_metadata(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """Compose the metadata payload sent with the evaluation request."""
+    base_metadata: Dict[str, Any] = {
+        "model": kwargs.get("model", MODEL_NAME),
+        "experiment": kwargs.get("experiment", EXPERIMENT_NAME),
+        "step_number": kwargs.get("step_number"),
+        "user_prompt": kwargs.get("user_prompt", USER_PROMPT_VERSION),
+        "batch_number": kwargs.get("batch_number"),
+        "prompt_index": kwargs.get("prompt_index"),
+        "rollout_group": kwargs.get("rollout_group"),
+        "system_prompt": kwargs.get("system_prompt", SYSTEM_PROMPT_VERSION),
+    }
+
+    step_metadata = kwargs.get("metadata") or {}
+    if isinstance(step_metadata, dict):
+        # Map common harness metadata onto our schema when present
+        step_value = _coerce_step_value(
+            step_metadata.get("step") or step_metadata.get("step_number")
+        )
+        if step_value is not None:
+            base_metadata["step_number"] = step_value
+
+        rollout_value = step_metadata.get("rollout_group")
+        if rollout_value is not None:
+            base_metadata["rollout_group"] = rollout_value
+
+        extras = step_metadata.get("extras")
+        if extras:
+            base_metadata["extras"] = extras
+
+        # Preserve any additional custom metadata fields
+        for key, value in step_metadata.items():
+            if key in {"step", "step_number", "rollout_group", "extras"}:
+                continue
+            if key not in base_metadata or base_metadata.get(key) is None:
+                base_metadata[key] = value
+
+    # Strip keys that remain None so the JSON is clean
+    return {key: value for key, value in base_metadata.items() if value is not None}
+
+
+def reward_fn(
+    completion: str,
+    **kwargs,
+) -> float:
+    """Evaluate the model response and return a reward score (0.0-1.0)."""
+    run_type = kwargs.get("run_type", RUN_TYPE)
+    run_version = kwargs.get("run_version", RUN_VERSION)
+    metadata = _build_metadata(kwargs)
+
+    payload = {
+        "code": completion,
+        "run_type": run_type,
+        "run_version": run_version,
+        "metadata": metadata,
+    }
+
+    try:
+        response = requests.post(
+            "https://eames-judge-api-769874896543.us-central1.run.app/evaluations-human",
+            json=payload,
+            timeout=1800,  # 30 minute timeout for screenshot generation
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        logger.info("Evaluation complete:")
+        logger.info("  Score: %s", result.get("score", 0.0))
+        logger.info("  Feedback: %s", result.get("explanation", "N/A"))
+        logger.info("  Processing Time (ms): %s", result.get("processing_time_ms", "N/A"))
+        logger.info("  Worker ID: %s", result.get("worker_id", "N/A"))
+        logger.info("  Success: %s", result.get("success", False))
+        if metadata:
+            logger.info("  Metadata sent: %s", metadata)
+
+        screenshot_urls = result.get("screenshot_urls", {}) or {}
+        if screenshot_urls:
+            logger.info("  Screenshot URLs:")
+            for key, url in screenshot_urls.items():
+                logger.info("    %s: %s", key.capitalize(), url)
+
+        score = result.get("score", 0.0)
+        if not isinstance(score, (int, float)):
+            logger.warning("Invalid score type: %s. Defaulting to 0.0", type(score))
+            return 0.0
+
+        return max(0.0, min(1.0, float(score)))
+
+    except requests.exceptions.Timeout:
+        logger.error("Request to evaluation server timed out")
+        return 0.0
+
+    except requests.exceptions.RequestException as exc:
+        logger.error("Request to evaluation server failed: %s", exc)
+        return 0.0
+
+    except (KeyError, ValueError, TypeError) as exc:
+        logger.error("Error parsing evaluation server response: %s", exc)
+        return 0.0
+
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.error("Unexpected error in reward_fn: %s", exc)
+        return 0.0

examples/agora_ex/task_app/system_prompt_CURRENT.md

@@ -0,0 +1,63 @@
+You are a powerful agentic AI coding assistant called Eames working with a Next.js + Shadcn/UI TypeScript project to generate exactly one complete landing page file for SaaS/Software products.
+
+## OUTPUT FORMAT
+
+Return ONLY a single TypeScript React component file.
+- Wrap in one code fence: ```tsx ... ```
+- No explanations, no additional text before or after the fence
+- If you cannot generate valid code, return nothing
+
+## FILE SPECIFICATION
+
+File: app/page.tsx
+Length: Target 800-1200 lines (flexible based on content richness)
+Tech: Next.js 14 App Router, TypeScript, React, Tailwind CSS only
+**IMPORTANT:** DO NOT use images, image imports, or next/image. Use SVG icons, Tailwind patterns, gradients, or CSS shapes instead.
+
+## REQUIRED STRUCTURE (in this exact order)
+
+1. Imports: NONE (do not import next/image or any other libraries)
+
+2. Metadata export:
+   ```tsx
+   export const metadata = {
+     title: "Page Title (max 80 chars)",
+     description: "Page description (max 160 chars)"
+   }
+   ```
+3. Helper components (if needed): Define small inline components AFTER the Page export
+   - Examples: FeatureCard, PricingCard, TestimonialCard, LogoItem
+   - Keep minimal, no deep nesting
+
+## DESIGN GUIDELINES
+Ship something interesting rather than boring, but never ugly.
+Include images and SVGs that are relevant to the category of business.
+
+## TECHNICAL CONSTRAINTS
+
+✓ Server component by default (no "use client" unless interactive state/events needed)
+✓ Tailwind utility classes for ALL styling
+✓ Semantic HTML5: `<main>`, `<section>`, `<header>`, `<footer>`, `<h1>`-`<h6>` hierarchy
+✓ Use inline SVG for icons (simple shapes: circles, squares, arrows, checkmarks, etc.)
+✓ Use Tailwind gradients, borders, and shadows for visual elements
+✓ Use CSS shapes and patterns instead of images
+
+✗ NO images - do not use `<img>`, `<Image>`, or any image imports
+✗ NO next/image imports
+✗ No data fetching (fetch, axios, server actions)
+✗ No lorem ipsum - write real, specific copy
+
+## VALIDATION
+
+Your output must:
+1. Be a single valid .tsx file with NO imports whatsoever
+2. Include `export const metadata`
+3. Include `export default function Page()`
+4. Include all 8 required sections in order: Navbar, Hero, Logos, Features, Testimonials, Pricing, Final CTA, Footer
+5. Use only Tailwind for styling
+6. Be deployable in Next.js 14 App Router without errors
+7. Use proper TypeScript syntax
+8. Follow the specific product category requested in the user prompt
+9. **NO images** - use SVG icons and Tailwind styling only
+
+Given the user's prompt describing the website theme/product, generate the code immediately.

examples/multi_step/configs/crafter_rl_outcome.toml

@@ -0,0 +1,74 @@
+# Crafter RL experiment – outcome rewards only (step rewards disabled)
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-outcome"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 12
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 5
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 20
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 8
+batch_size = 3
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+step_rewards_enabled = false
+event_rewards_kind = "unique"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0

examples/multi_step/configs/crafter_rl_stepwise_hosted_judge.toml

@@ -0,0 +1,175 @@
+# Crafter RL experiment – stepwise shaping with hosted judge rubrics
+#
+# This configuration extends the stepwise LoRA baseline by wiring the Synth judge
+# service so evaluation rolls combine dense step rewards with hosted rubric scoring.
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
+judge_url = "https://YOUR-BACKEND-ENDPOINT/api"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-hosted-judge"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+  [rollout.env_config]
+  difficulty = "easy"
+
+    [rollout.env_config.step_rewards]
+    enabled = true
+    mode = "decision_stepwise"
+    strategy = "consistent"       # +1 for each decision that unlocks a new achievement
+    indicator_lambda = 1.0
+    step_beta = 0.0
+
+  [rollout.policy_config]
+  temperature = 0.2
+  top_p = 0.95
+  max_tokens = 512
+
+[evaluation]
+instances = 16
+every_n_iters = 8
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 16
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+event_rewards_kind = "unique"
+
+# Enable dense decision rewards in the trainer to mirror env_config step rewards.
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 1.0
+step_rewards_beta = 0.0
+step_rewards_strategy = "consistent"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0
+
+[rubric]
+enabled = true
+# Blend the hosted judge scores with environment returns inside the trainer.
+[rubric.weights]
+env = 0.2
+event = 0.4
+outcome = 0.4
+
+[rubric.event]
+# Hosted judge rubric for per-decision progress scoring.
+rubric_id = "crafter/event@v1"
+
+[rubric.outcome]
+# Hosted judge rubric for final trajectory scoring.
+rubric_id = "crafter/outcome@v1"
+
+[judge]
+type = "gemini"                            # or "groq" when routing to Groq-hosted judges
+timeout_s = 45
+
+  [judge.options]
+  event = true
+  outcome = true
+  provider = "openai"
+  model = "openai/gpt-oss-120b"
+  rubric_id = "crafter/bundle@v1"
+  max_concurrency = 6
+  tracks = ["process", "reasoning", "progress", "outcome"]
+
+    [judge.options.rubric_overrides]
+
+      [judge.options.rubric_overrides.event]
+      goal_text = """
+Treat each decision as a check for new Crafter achievements.
+Award the top score only when the log shows a fresh achievement unlock or an immediately verifiable deterministic completion.
+Keep otherwise useful setup actions in a narrow low band so non-achievement turns stay near zero."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "progress.unique_achievements"
+        weight = 0.9
+        scale = "binary"
+        description = "Return 1 when this decision explicitly unlocks a brand-new Crafter achievement (inventory or status text confirms it this turn). Otherwise return 0."
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "process.intent_alignment"
+        weight = 0.1
+        scale = "bounded"
+        description = "Use at most 0.3 to acknowledge tightly coupled setup that finishes the last prerequisite; keep ≤0.1 when the agent only repositions or gathers without an imminent unlock."
+
+      [judge.options.rubric_overrides.outcome]
+      goal_text = """
+Summarise the episode outcome in relation to Crafter’s win condition:
+survive, accumulate resources, and craft advanced tools or structures.
+Highlight notable achievements, safety failures, and preparedness for future exploration."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.goal_completion"
+        weight = 0.6
+        scale = "binary"
+        description = "Full credit when the agent ends with strong survival metrics and a clear crafted milestone (e.g., iron tools, furnace)."
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.achievement_depth"
+        weight = 0.4
+        scale = "bounded"
+        description = "Partial credit for intermediate achievements (saplings, wood/stone tools) that set up future success."
+
+    [judge.options.weights]
+    process = 0.05
+    reasoning = 0.15
+    progress = 0.30
+    outcome = 0.50

examples/multi_step/configs/crafter_rl_stepwise_shaped.toml

@@ -0,0 +1,83 @@
+# Crafter RL experiment – shaped stepwise rewards (achievement + resource shaping)
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-shaped"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 10
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 0.5
+step_rewards_beta = 0.0
+event_rewards_kind = "unique"
+step_rewards_strategy = "per_achievement"
+
+# Reward each achievement up to a cap inside `compute_stepwise_reward`
+step_rewards_weights = { collect_sapling = 0.6, collect_wood = 0.8, collect_stone = 1.0, collect_iron = 1.2, collect_drink = 0.4, collect_food = 0.4 }
+step_rewards_k_limits = { collect_sapling = 2, collect_wood = 4, collect_stone = 3, collect_iron = 3, collect_drink = 3, collect_food = 3 }
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0

examples/multi_step/configs/crafter_rl_stepwise_simple.toml

@@ -0,0 +1,78 @@
+# Crafter RL experiment – simple stepwise rewards (1 point per new achievement)
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-simple"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 10
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 1.0
+step_rewards_beta = 0.0
+event_rewards_kind = "unique"
+step_rewards_strategy = "consistent"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0

examples/multi_step/crafter_rl_lora.md

@@ -2,28 +2,69 @@
 
 This walkthrough shows how to fine-tune the Crafter task app with our 10-step RL LoRA config.
 
-1. **Start the Crafter task app on Modal (with tracing + text-only prompts)**
+1. **Deploy the Crafter task app on Modal**
 
    ```bash
-   BACKEND_BASE_URL=https://agent-learning.onrender.com/api \
+   # assumes .env contains SYNTH_API_KEY, ENVIRONMENT_API_KEY, GROQ_API_KEY, etc.
    uvx synth-ai modal-serve grpo-crafter \
      --env-file examples/warming_up_to_rl/.env \
      --name grpo-crafter-task-app
    ```
 
-   * Deploys the Modal task app with the tracing/text-only fixes baked in.*
+   * The command prints the public `https://…modal.run` URL; copy it for the RL configs below.*
 
-2. **Launch the RL job using the updated LoRA config**
+2. **Wire up the three RL experiment configs**
+
+   Update the `task_url` placeholder in each config with the Modal URL from step 1:
+
+   - `examples/multi_step/configs/crafter_rl_outcome.toml`
+   - `examples/multi_step/configs/crafter_rl_stepwise_simple.toml`
+   - `examples/multi_step/configs/crafter_rl_stepwise_shaped.toml`
+
+   The difference between them (all run with LoRA on 2×H100 split 1/1 for vLLM vs. trainer):
+
+   | Config | Reward signal |
+   | ------ | ------------- |
+   | `crafter_rl_outcome.toml` | Outcome-only — step rewards disabled. |
+   | `crafter_rl_stepwise_simple.toml` | Stepwise (“consistent”) — +1 for every newly unlocked achievement. |
+   | `crafter_rl_stepwise_shaped.toml` | Stepwise (“per_achievement”) — combines achievement credit with inventory/achievement-count shaping from the rollout hook. |
+
+3. **Launch the three RL runs in parallel**
 
    ```bash
+   export SYNTH_API_KEY=...       # already sourced if examples/.env was loaded
+   export TASK_APP_URL=https://your-modal-task-app.modal.run
+
+   uvx synth-ai train --type rl \
+     --config examples/multi_step/configs/crafter_rl_outcome.toml \
+     --run-name crafter-rl-outcome \
+     --no-poll &
+
    uvx synth-ai train --type rl \
-     --config tests/artifacts/configs/rl.lora.small.toml \
-     --backend https://agent-learning.onrender.com/api \
-     --env-file .env \
-     --no-poll
+     --config examples/multi_step/configs/crafter_rl_stepwise_simple.toml \
+     --run-name crafter-rl-stepwise-simple \
+     --no-poll &
+
+   uvx synth-ai train --type rl \
+     --config examples/multi_step/configs/crafter_rl_stepwise_shaped.toml \
+     --run-name crafter-rl-stepwise-shaped \
+     --no-poll &
+
+   wait
    ```
 
-   * This config forces 10 agent turns per rollout, reduces batch size to avoid OOMs, and enforces Crafter-specific defaults.*
+   *`--no-poll` returns immediately so each run can stream logs in its own terminal; `wait` blocks until all jobs finish.*
+
+4. **Track results**
+
+   Tail each job’s logs with `uvx synth-ai train logs --run-name <name>` or open the Modal dashboard. Compare:
+
+   - Avg outcome reward (modal dashboard)
+   - Stepwise reward components (`resource_reward`, `unique_achievements_total`) in the task app logs
+   - Trace JSONL dumps under `traces/v3` if tracing is enabled
+
+
+  * This config forces 10 agent turns per rollout, reduces batch size to avoid OOMs, and enforces Crafter-specific defaults.*
 
   INFO - 🎉 Training completed successfully!
-  INFO - All batch rewards: [0.0625, 0.0625, 0.125, 0.0625, 0.0625, 0.3125, 0.375, 0.4375, 0.5, 0.9375]
+  INFO - All batch rewards: [0.0625, 0.0625, 0.125, 0.0625, 0.0625, 0.3125, 0.375, 0.4375, 0.5, 0.9375]