synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

examples/qwen_coder/validate_jsonl.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-"""Validate that a JSONL file parses and contains chat-like records.
-
-Checks first N lines (default 50) for objects with `messages` including an
-assistant response (role == "assistant").
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-
-
-def main() -> None:
-    p = argparse.ArgumentParser(description=__doc__)
-    p.add_argument("path", help="Path to JSONL file")
-    p.add_argument("--n", type=int, default=50, help="Number of lines to sample")
-    args = p.parse_args()
-
-    src = Path(args.path)
-    if not src.exists():
-        raise SystemExit(f"No such file: {src}")
-
-    checked = 0
-    ok = 0
-    with src.open("r", encoding="utf-8") as f:
-        for i, line in enumerate(f, start=1):
-            if i > args.n:
-                break
-            line = line.strip()
-            if not line:
-                continue
-            checked += 1
-            try:
-                obj = json.loads(line)
-            except Exception as exc:
-                raise SystemExit(f"Line {i} is not valid JSON: {exc}")
-            msgs = obj.get("messages") if isinstance(obj, dict) else None
-            if not isinstance(msgs, list):
-                raise SystemExit(f"Line {i} missing 'messages' list")
-            has_assistant = any(
-                isinstance(m, dict) and m.get("role") == "assistant" and m.get("content")
-                for m in msgs
-            )
-            if has_assistant:
-                ok += 1
-
-    if checked == 0:
-        raise SystemExit("No lines checked; file empty?")
-    if ok == 0:
-        raise SystemExit("No assistant messages found in sampled lines")
-    print(f"Validated: {ok}/{checked} sampled lines contain assistant messages")
-
-
-if __name__ == "__main__":
-    main()
-
-
-

examples/rl/README.md

@@ -1,169 +0,0 @@
-# Math RL Demo (Single Step)
-
-This example trains a reinforcement learning policy on single-step math problems sourced from the [EleutherAI/math](https://huggingface.co/datasets/EleutherAI/math) dataset. Episodes consist of a single tool call: the model must emit a `math_submit` function call whose `answer` field contains the final solution. Missing or malformed tool calls receive negative reward; correct answers earn positive reward.
-
-## Quick Commands
-
-```bash
-# Serve locally with tracing
-uvx synth-ai serve math-single-step --port 8101 --env-file examples/rl/.env --trace traces/math
-
-# Modal deployment
-uvx synth-ai deploy --name synth-math-single-step --env-file examples/rl/.env
-
-# Evaluate base Qwen policy (validation split)
-uv run python examples/rl/run_eval.py --toml examples/rl/configs/eval_base_qwen.toml
-
-# Launch RL job from base model
-uvx synth-ai train --type rl --config examples/rl/configs/rl_from_base_qwen.toml
-
-# Evaluate RL checkpoint on held-out test split
-uv run python examples/rl/run_eval.py --toml examples/rl/configs/eval_rl_qwen.toml
-```
-
-## 1. Prerequisites
-
-- Python 3.11+
-- `uv`/`uvx`
-- Modal CLI (`modal token new`) for deployment
-- `.env` at `examples/rl/.env` containing at least:
-  - `SYNTH_API_KEY`
-  - `ENVIRONMENT_API_KEY`
-  - Optional: `TASK_APP_URL` (Modal URL), `GROQ_API_KEY`, `OPENAI_API_KEY`
-
-Run `uvx synth-ai setup` to populate the `.env` if you have not paired the SDK before.
-
-## 2. Task App
-
-The task app is defined in `synth_ai/task/apps/math_single_step.py` and registered as `math-single-step`. It loads problems from the Hugging Face dataset (configurable via `MATH_DATASET_*` env vars) and manages per-episode state with an in-memory environment manager.
-
-- **Observation**: single math problem (string) plus dataset metadata.
-- **Actions**: exactly one `math_submit` tool call with an `answer` string.
-- **Rewards**:
-  - `+1.0` for correct answer
-  - `0.0` for incorrect answer
-  - `-0.5` if the tool call omits an answer or uses the wrong tool
-  - `-1.0` when no tool call is provided
-
-Serve locally with tracing to capture trajectories:
-
-```bash
-uvx synth-ai serve math-single-step \
-  --port 8101 \
-  --env-file examples/rl/.env \
-  --trace traces/math \
-  --trace-db traces/math/synth_ai.db
-```
-
-Deploy or serve on Modal using the same env file; the registration includes a `ModalDeploymentConfig` that installs the `datasets` package automatically.
-
-## 3. Evaluation
-
-`examples/rl/run_eval.py` evaluates a policy by sampling deterministic seeds from the dataset splits. TOML configuration controls the model, split, and number of episodes. Example config (`eval_base_qwen.toml`):
-
-```toml
-provider = "synth"
-task_app_url = "http://localhost:8101"
-model = "Qwen/Qwen3-4B"
-split = "validation"
-num_episodes = 50
-seed_start = 0
-
-[policy]
-inference_url = "http://localhost:8000/api/inference"
-max_tokens = 128
-temperature = 0.0
-# Optional: override headers for inference requests
-# [policy.extra_headers]
-# Authorization = "Bearer ..."
-```
-
-The `[policy]` table maps directly to the inference payload; add `[policy.headers]` if you need to forward custom HTTP headers (e.g., `Authorization`). If `SYNTH_API_KEY` is present, the evaluator automatically sends `Authorization: Bearer <key>`.
-
-Set `--use-rollout` to exercise the server-side rollout endpoint instead of the per-step API.
-
-The script reports accuracy and a breakdown of failure modes (`missing_tool_call`, `blank_answer`, etc.).
-
-## 4. RL Training
-
-Example RL config (`configs/rl_from_base_qwen.toml`):
-
-```toml
-[services]
-task_url = "https://your-app.modal.run"
-
-[model]
-base = "Qwen/Qwen3-4B"
-
-[data]
-split = "train"
-seed_start = 0
-episodes_per_iteration = 2048
-
-[training]
-max_turns = 1
-ops = ["agent", "env"]
-batch_size = 128
-group_size = 1024
-reward_positive = 1.0
-reward_negative_no_tool = -1.0
-reward_negative_no_answer = -0.5
-
-[policy]
-model = "Qwen/Qwen3-4B"
-inference_url = "https://your-inference-host"
-max_tokens = 128
-temperature = 0.0
-
-[tags]
-experiment = "math_single_step"
-```
-
-Submit jobs interactively with:
-
-```bash
-uvx synth-ai train --type rl --config examples/rl/configs/rl_from_base_qwen.toml
-```
-
-The CLI ensures the task app is reachable (`/health`, `/task_info`), prompts for missing secrets, and polls job status until completion. For scripted automation, use `run_rl_and_save.py`:
-
-```bash
-uv run python examples/rl/run_rl_and_save.py \
-  --config examples/rl/configs/rl_from_base_qwen.toml \
-  --backend https://backend.synth.ai/api
-```
-
-## 5. Evaluating RL Outputs
-
-After training completes, set `model = "rl:<job_or_model_id>"` in `configs/eval_rl_qwen.toml` (and update `split = "test"` for a held-out set). Re-run `run_eval.py` to compare:
-
-```bash
-uv run python examples/rl/run_eval.py --toml examples/rl/configs/eval_rl_qwen.toml
-```
-
-Record both validation (pre-RL) and test (post-RL) accuracy to quantify improvements.
-
-## 6. Dataset Notes
-
-- By default the task app loads the [Hendrycks MATH benchmark](https://huggingface.co/datasets/nlile/hendrycks-MATH-benchmark). Override via `MATH_DATASET_NAME` / `MATH_DATASET_CONFIG` env vars if you want a different variant. The dataset is public and automatically downloaded when the task app starts; the server will fail fast with a clear error if it cannot be fetched.
-- For offline use, run `uv run python examples/rl/download_dataset.py --output-dir examples/rl/data --dataset nlile/hendrycks-MATH-benchmark --config algebra --limit 2000`. Then start the task app with `MATH_DATASET_LOCAL_DIR=examples/rl/data` (or set `MATH_DATASET_LOCAL_<SPLIT>_FILE`).
-- Hugging Face downloads occur at runtime; pre-fetch locally or mount a Modal volume if you need offline access.
-- Hugging Face downloads occur at runtime; pre-fetch locally or mount a Modal volume if you need offline access.
-- Seeds map directly to dataset indices. Use `seed_start` to control determinism in configs and evaluations.
-
-## 7. Additional Utilities
-
-- `examples/rl/task_app/math_task_app.py` – legacy runner (`python .../math_task_app.py --reload`).
-- `examples/rl/run_eval.py` – CLI evaluation helper (supports proxying Groq or hitting arbitrary inference URLs).
-- `examples/rl/run_rl_and_save.py` – thin wrapper around the Synth `/rl/jobs` API.
-
-For broader background on Synth task apps, CLI commands, and tracing, see the new documentation under `docs/`.
-
-
-
-uv run python examples/rl/run_eval.py --toml examples/rl/configs/eval_base_qwen.toml
-uvx synth-ai serve math-single-step \
-    --port 8101 \
-    --env-file examples/rl/.env \
-    --trace traces/math \
-    --force

examples/rl/download_dataset.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-"""Download subsets of the MATH dataset to local JSONL files."""
-
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-from typing import Any
-
-from datasets import load_dataset
-
-
-def extract_examples(dataset: Any, *, limit: int | None) -> list[dict[str, str]]:
-    if limit is not None:
-        dataset = dataset.select(range(min(limit, len(dataset))))
-    examples: list[dict[str, str]] = []
-    for item in dataset:
-        problem = (item.get("problem") or "").strip()
-        solution = item.get("solution") or ""
-        if isinstance(solution, list):
-            solution = "\n".join(str(part) for part in solution)
-        examples.append(
-            {
-                "problem": problem,
-                "solution": solution,
-            }
-        )
-    return examples
-
-
-def write_jsonl(path: Path, rows: list[dict[str, str]]) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with path.open("w", encoding="utf-8") as fh:
-        for row in rows:
-            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Download MATH dataset splits to JSONL for offline use"
-    )
-    parser.add_argument(
-        "--output-dir", default="examples/rl/data", help="Directory to write <split>.jsonl files"
-    )
-    parser.add_argument(
-        "--dataset",
-        default="nlile/hendrycks-MATH-benchmark",
-        help="Hugging Face dataset identifier",
-    )
-    parser.add_argument(
-        "--config", default="algebra", help="Hugging Face dataset config (if required)"
-    )
-    parser.add_argument(
-        "--splits", nargs="*", default=["train", "validation", "test"], help="Splits to download"
-    )
-    parser.add_argument(
-        "--limit", type=int, default=None, help="Optional cap on examples per split"
-    )
-    args = parser.parse_args()
-
-    output_dir = Path(args.output_dir).expanduser()
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    for split in args.splits:
-        print(f"[INFO] Downloading {args.dataset} ({args.config}) split={split}")
-        if args.config:
-            dataset = load_dataset(args.dataset, args.config, split=split)
-        else:
-            dataset = load_dataset(args.dataset, split=split)
-        rows = extract_examples(dataset, limit=args.limit)
-        out_path = output_dir / f"{split}.jsonl"
-        write_jsonl(out_path, rows)
-        print(f"[INFO] Wrote {len(rows)} examples to {out_path}")
-
-    print("Done. Set MATH_DATASET_LOCAL_DIR to the output directory when serving the task app.")
-
-
-if __name__ == "__main__":
-    main()

examples/run_crafter_demo.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Run a Crafter agent demo with Gemini
-# This script demonstrates a reactive agent in the Crafter environment
-
-echo "🚀 Starting Crafter agent demo with Gemini 1.5 Flash..."
-echo "Make sure the synth-ai service is running: uvx synth-ai serve"
-echo ""
-
-uv run python -m synth_ai.environments.examples.crafter_classic.agent_demos.test_crafter_react_agent --model gemini-1.5-flash

examples/sft/README.md

@@ -1,139 +0,0 @@
-### Supervised Fine-Tuning for Crafter
-
-This folder provides a minimal, reusable SFT workflow that pulls out the SFT step from `examples/warming_up_to_rl/` and focuses it on LoRA/QLoRA. We've also added guidance for running full finetuning (FFT) so you can compare adapters against end-to-end weight updates.
-
-It supports distilling Groq (or other vendor) rollouts into JSONL using tracing and then training a small base model like `Qwen/Qwen3-0.6B`.
-
----
-
-### 0) Load environment from .env.dev (recommended)
-
-Use your dev env file so keys/URLs are sourced consistently:
-
-```bash
-# Example path; update to your actual dev env
-set -a && source /Users/joshpurtell/Documents/GitHub/monorepo/backend/.env.dev && set +a
-```
-
-This ensures `ENVIRONMENT_API_KEY`, `GROQ_API_KEY`, and (optionally) `BACKEND_BASE_URL` are available to the steps below.
-
----
-
-### 1) Collect traces and export SFT JSONL
-
-You can generate traces with the Crafter task app and then export them to SFT JSONL using the existing exporter:
-
-```bash
-# Serve the task app locally with tracing enabled (example)
-uvx synth-ai serve grpo-crafter \
-  --trace traces/v3 \
-  --trace-db traces/v3/synth_ai.db \
-  --port 8001
-
-# Or run traced local rollouts to accumulate data
-uv run python examples/warming_up_to_rl/run_local_rollout_traced.py \
-  --episodes 50 --max-turns 10
-
-# Export SFT dataset from the trace DB
-uv run python examples/warming_up_to_rl/export_trace_sft.py \
-  --db traces/v3/synth_ai.db \
-  --min-unique 0 \
-  --output examples/sft/ft_data/crafter_traces.jsonl
-```
-
-Notes:
-- The exporter uses achievements and event rewards to filter high-signal steps. Combine `--min-unique`, `--min-outcome-reward`, `--event-reward`, and `--require-achievement` to control data quality.
-- You can restrict to sessions from certain providers/models with `--provider`/`--model`.
-- Use `--limit` while debugging to reduce dataset size quickly.
-
----
-
-### 2a) Train LoRA (QLoRA) on Qwen/Qwen3-0.6B
-
-Use the standard CLI. Do not use a custom Python finetuning script. Point the CLI at your `.env.dev` so it picks up keys automatically:
-
-```bash
-uvx synth-ai train \
-  --type sft \
-  --config examples/sft/configs/crafter_lora_qwen0p6b.toml \
-  --dataset examples/sft/ft_data/crafter_traces.jsonl \
-  --env-file /Users/joshpurtell/Documents/GitHub/monorepo/backend/.env.dev
-```
-
-The config sets `training.use_qlora = true` and `hyperparameters.train_kind = "peft"` to request LoRA adapters.
-
-Experiment tips:
-- The backend currently defaults to a LoRA rank of 16. If you need other ranks, generate the payload with `--dry-run`, add `"lora_rank": <value>` (and optional `"lora_alpha"`, `"lora_dropout"`) under `hyperparameters`, and submit it via the API until the CLI exposes these knobs directly.
-- Duplicate the TOML and adjust `hyperparameters.warmup_ratio`, `learning_rate`, or `gradient_accumulation_steps` to keep the global batch size comparable across datasets.
-
----
-
-### 2b) Train Full Finetune (FFT) on Qwen/Qwen3-0.6B
-
-Full finetuning updates all weights and uses a near-identical CLI flow with the LoRA toggle disabled. The helper config lives alongside the LoRA sample:
-
-```bash
-uvx synth-ai train \
-  --type sft \
-  --config examples/sft/configs/crafter_fft_qwen0p6b.toml \
-  --dataset examples/sft/ft_data/crafter_traces.jsonl \
-  --env-file /Users/joshpurtell/Documents/GitHub/monorepo/backend/.env.dev
-```
-
-Key differences vs LoRA:
-- `training.use_qlora = false` and `hyperparameters.train_kind = "fft"` request a full-weight update.
-- `per_device_batch` defaults to 1 to keep memory use comfortable on a single H100; raise gradually as you confirm headroom.
-- FFT runs slower per step. Consider trimming the dataset with `--examples` or the exporter filters for quick baselines.
-
-If you want the 4B Crafter FFT baseline from the RL examples, reuse `examples/warming_up_to_rl/configs/crafter_fft_4b.toml` with the same CLI command.
-
----
-
-### 3) Evaluate the fine-tuned models
-
-After the job completes, list your fine-tuned models and evaluate them in the Crafter loop:
-
-```bash
-# List models
-uv run python - <<'PY'
-import asyncio
-import os
-from synth_ai.learning.client import LearningClient
-
-backend = os.getenv("BACKEND_BASE_URL", "https://agent-learning.onrender.com/api")
-api_key = os.getenv("SYNTH_API_KEY", "")
-async def main():
-    client = LearningClient(backend, api_key)
-    models = await client.list_fine_tuned_models()
-    for m in models:
-        print(m)
-asyncio.run(main())
-PY
-
-# Evaluate in the Crafter eval loop (example via warming_up_to_rl)
-TASK_APP_URL=http://localhost:8001 \
-uv run python examples/warming_up_to_rl/run_eval.py \
-  --toml examples/warming_up_to_rl/configs/eval_local_vllm.toml \
-  --model ft:YOUR_FT_MODEL_ID \
-  --use-rollout
-```
-
----
-
-### 4) Plan comparison runs
-
-Keep runs comparable by adjusting one axis at a time and logging the settings in your experiment tracker (spreadsheet, weights & biases, etc.).
-
-- **LoRA rank sweeps:** start from `crafter_lora_qwen0p6b.toml`, clone it per rank (e.g., `r=4,8,16,64`). For now add the desired `lora_rank` in the job payload manually (see note above) and include it in the run name.
-- **Dataset size:** duplicate the exported JSONL and slice with `head -n`, or pass `--examples N` to the CLI for quick subsamples. Track the effective token count using the exporter logs.
-- **Data quality:** increase `--min-unique`, require specific achievements, or exclude low-reward sessions with `export_trace_sft.py`. Capture the filter tuple in your run metadata so evaluations stay reproducible.
-- **FFT vs LoRA:** run both configs on the same dataset/cardinality so differences reflect the training method rather than the data.
-
-For each sweep, use consistent evaluation seeds and write down throughput (tokens/sec) so you can weigh quality vs cost.
-
----
-
-### Files
-- `configs/crafter_lora_qwen0p6b.toml`: LoRA/QLoRA SFT config for `Qwen/Qwen3-0.6B`.
-- `configs/crafter_fft_qwen0p6b.toml`: Full-finetune SFT config for `Qwen/Qwen3-0.6B`.
-- `ft_data/`: place your exported JSONL here (ignored by VCS).

examples/sft/configs/crafter_fft_qwen0p6b.toml

@@ -1,44 +0,0 @@
-[job]
-model = "Qwen/Qwen3-0.6B"
-# Prefer passing --dataset at runtime for repeatability
-# data = "examples/sft/ft_data/crafter_traces.jsonl"
-
-[compute]
-gpu_type = "H100"
-gpu_count = 1
-nodes = 1
-
-[data]
-topology = {}
-# Optional validation set if you have one locally
-# validation_path = "examples/sft/ft_data/crafter_traces.val.jsonl"
-
-[training]
-mode = "sft_offline"
-use_qlora = false
-
-[training.validation]
-enabled = true
-evaluation_strategy = "steps"
-eval_steps = 50
-save_best_model_at_end = true
-metric_for_best_model = "val.loss"
-greater_is_better = false
-
-[hyperparameters]
-n_epochs = 1
-train_kind = "fft"
-per_device_batch = 1
-gradient_accumulation_steps = 32
-sequence_length = 4096
-learning_rate = 1e-5
-warmup_ratio = 0.03
-weight_decay = 0.01
-
-[hyperparameters.parallelism]
-use_deepspeed = true
-deepspeed_stage = 2
-fsdp = false
-bf16 = true
-fp16 = false
-activation_checkpointing = true

examples/sft/configs/crafter_lora_qwen0p6b.toml

@@ -1,45 +0,0 @@
-[job]
-model = "Qwen/Qwen3-0.6B"
-# Optionally set here, but prefer passing --dataset at runtime
-# data = "examples/sft/ft_data/crafter_traces.jsonl"
-
-[compute]
-gpu_type = "H100"
-gpu_count = 1
-nodes = 1
-
-[data]
-# Forwarded into metadata.effective_config
-topology = {}
-# Optional validation set if you have one locally
-# validation_path = "examples/sft/ft_data/crafter_traces.val.jsonl"
-
-[training]
-mode = "lora"
-use_qlora = true
-
-[training.validation]
-enabled = true
-evaluation_strategy = "steps"
-eval_steps = 50
-save_best_model_at_end = true
-metric_for_best_model = "val.loss"
-greater_is_better = false
-
-[hyperparameters]
-n_epochs = 1
-train_kind = "peft"
-per_device_batch = 2
-gradient_accumulation_steps = 32
-sequence_length = 4096
-learning_rate = 5e-6
-warmup_ratio = 0.03
-
-[hyperparameters.parallelism]
-use_deepspeed = true
-deepspeed_stage = 2
-fsdp = false
-bf16 = true
-fp16 = false
-activation_checkpointing = true
-

examples/sft/evaluate.py

@@ -1,119 +0,0 @@
-#!/usr/bin/env python3
-"""Evaluate a base or fine-tuned model on Crafter via the Task App rollout.
-
-This mirrors the minimal evaluation loop: call `/rollout` for a set of seeds
-and report outcome/step metrics. If tracing is enabled server-side, you can
-use the exported sqlite DB for further analysis.
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import os
-from dataclasses import dataclass
-from typing import Any
-
-from synth_ai.task import (
-    RolloutEnvSpec,
-    RolloutPolicySpec,
-    RolloutRecordConfig,
-    RolloutRequest,
-    TaskAppClient,
-)
-
-
-@dataclass(slots=True)
-class EvalArgs:
-    base_url: str
-    api_key: str
-    model: str
-    inference_url: str
-    inference_api_key: str
-    seeds: list[int]
-    max_llm_calls: int
-    timeout: float
-
-
-def _ops(n: int) -> list[str]:
-    n = max(1, n)
-    ops: list[str] = []
-    for _ in range(n):
-        ops.extend(["agent", "env"])  # one LLM step followed by one env step
-    return ops
-
-
-def _request(seed: int, a: EvalArgs) -> RolloutRequest:
-    from synth_ai.task.contracts import RolloutMode
-    return RolloutRequest(
-        run_id=f"eval-{seed}",
-        env=RolloutEnvSpec(env_name="crafter", seed=seed, config={}),
-        policy=RolloutPolicySpec(
-            policy_name="crafter-react",
-            config={"model": a.model, "inference_url": a.inference_url, "api_key": a.inference_api_key},
-        ),
-        ops=_ops(a.max_llm_calls),
-        record=RolloutRecordConfig(trajectories=True, return_trace=False, trace_format="compact"),
-        mode=RolloutMode.EVAL,
-    )
-
-
-async def _eval_seed(client: TaskAppClient, seed: int, a: EvalArgs) -> dict[str, Any]:
-    resp = await client.rollout(_request(seed, a))
-    m = resp.metrics
-    return {"seed": seed, "num_steps": m.num_steps, "episode_returns": m.episode_returns, "outcome_score": m.outcome_score}
-
-
-async def main() -> None:
-    p = argparse.ArgumentParser(description=__doc__)
-    p.add_argument("--base-url", default=os.getenv("TASK_APP_URL", "http://localhost:8001"))
-    p.add_argument("--api-key", default=os.getenv("ENVIRONMENT_API_KEY"))
-    p.add_argument("--model", required=True, help="Base or ft:<id> to evaluate")
-    p.add_argument("--inference-url", default=os.getenv("INFERENCE_URL", "https://api.groq.com/openai"))
-    p.add_argument("--inference-api-key", default=os.getenv("GROQ_API_KEY"))
-    p.add_argument("--seeds", default="0,1,2,3,4,5,6,7,8,9")
-    p.add_argument("--max-llm-calls", type=int, default=10)
-    p.add_argument("--timeout", type=float, default=60.0)
-    args = p.parse_args()
-
-    seeds = [int(s) for s in str(args.seeds).split(",") if s.strip()]
-    a = EvalArgs(
-        base_url=str(args.base_url).strip(),
-        api_key=str(args.api_key or "").strip(),
-        model=str(args.model).strip(),
-        inference_url=str(args.inference_url).strip(),
-        inference_api_key=str(args.inference_api_key or "").strip(),
-        seeds=seeds,
-        max_llm_calls=int(args.max_llm_calls),
-        timeout=float(args.timeout),
-    )
-    if not a.api_key:
-        raise SystemExit("ENVIRONMENT_API_KEY is required")
-    if not a.inference_api_key:
-        raise SystemExit("Inference API key (e.g., GROQ_API_KEY) is required")
-
-    results: list[dict[str, Any]] = []
-    async with TaskAppClient(a.base_url, api_key=a.api_key, timeout=a.timeout) as client:
-        for seed in a.seeds:
-            r = await _eval_seed(client, seed, a)
-            results.append(r)
-            print(f"seed={seed} return={r.get('episode_returns')}")
-
-    # Simple aggregate
-    flat_returns: list[float] = []
-    for r in results:
-        ers = r.get("episode_returns") or []
-        if isinstance(ers, list) and ers:
-            try:
-                flat_returns.append(float(ers[0]))
-            except Exception:
-                pass
-    if flat_returns:
-        mean_ret = sum(flat_returns) / len(flat_returns)
-        print(f"mean_return={mean_ret:.3f} over {len(flat_returns)} episodes")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
-
-