synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

examples/sft/generate_traces.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-"""Generate Crafter rollouts and server-side traces via the Task App.
-
-This script is a slim wrapper around the Task App `/rollout` endpoint to
-produce trajectories while the server (if configured with TASKAPP_TRACING_ENABLED)
-persists traces to its sqlite database. Use `export_dataset.py` afterwards
-to build an SFT JSONL.
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import os
-import sys
-import time
-from dataclasses import dataclass
-from typing import Any
-
-from synth_ai.task import (
-    RolloutEnvSpec,
-    RolloutPolicySpec,
-    RolloutRecordConfig,
-    RolloutRequest,
-    TaskAppClient,
-)
-
-
-def _ensure_str(val: Any, name: str) -> str:
-    s = str(val or "").strip()
-    if not s:
-        raise SystemExit(f"Missing required {name}")
-    return s
-
-
-def _build_ops(max_llm_calls: int) -> list[str]:
-    max_llm_calls = max(1, int(max_llm_calls or 1))
-    ops: list[str] = []
-    for _ in range(max_llm_calls):
-        ops.extend(["agent", "env"])  # LLM step then env step
-    return ops
-
-
-def _build_request(seed: int, run_id: str, model: str, inference_url: str, api_key: str, *, max_llm_calls: int, return_trace: bool) -> RolloutRequest:
-    policy_cfg: dict[str, Any] = {
-        "model": model,
-        "inference_url": inference_url,
-        "api_key": api_key,
-    }
-    record = RolloutRecordConfig(trajectories=True, return_trace=bool(return_trace), trace_format="compact")
-    return RolloutRequest(
-        run_id=run_id,
-        env=RolloutEnvSpec(env_name="crafter", seed=seed, config={}),
-        policy=RolloutPolicySpec(policy_name="crafter-react", config=policy_cfg),
-        ops=_build_ops(max_llm_calls),
-        record=record,
-    )
-
-
-@dataclass(slots=True)
-class Args:
-    base_url: str
-    api_key: str
-    inference_url: str
-    inference_api_key: str
-    model: str
-    episodes: int
-    start_seed: int
-    max_llm_calls: int
-    concurrency: int
-    return_trace: bool
-    timeout: float
-
-
-async def _run_one(client: TaskAppClient, run_id: str, seed: int, a: Args) -> dict[str, Any]:
-    req = _build_request(
-        seed=seed,
-        run_id=f"{run_id}-seed{seed}",
-        model=a.model,
-        inference_url=a.inference_url,
-        api_key=a.inference_api_key,
-        max_llm_calls=a.max_llm_calls,
-        return_trace=a.return_trace,
-    )
-    resp = await client.rollout(req)
-    metrics = resp.metrics.model_dump()
-    return {
-        "seed": seed,
-        "num_steps": metrics.get("num_steps"),
-        "episode_returns": metrics.get("episode_returns"),
-        "outcome_score": metrics.get("outcome_score"),
-    }
-
-
-async def _bounded_gather(n: int, coros: list[asyncio.Future]):
-    sem = asyncio.Semaphore(n)
-
-    async def _wrap(coro):
-        async with sem:
-            return await coro
-
-    return await asyncio.gather(*[_wrap(c) for c in coros])
-
-
-async def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--base-url", default=os.getenv("TASK_APP_URL", "http://localhost:8001"))
-    parser.add_argument("--api-key", default=os.getenv("ENVIRONMENT_API_KEY"))
-    parser.add_argument("--inference-url", default=os.getenv("INFERENCE_URL", "https://api.groq.com/openai"))
-    parser.add_argument("--inference-api-key", default=os.getenv("GROQ_API_KEY"))
-    parser.add_argument("--model", default=os.getenv("POLICY_MODEL", "llama-3.3-70b-versatile"))
-    parser.add_argument("--episodes", type=int, default=50)
-    parser.add_argument("--start-seed", type=int, default=0)
-    parser.add_argument("--max-llm-calls", type=int, default=10)
-    parser.add_argument("--concurrency", type=int, default=5)
-    parser.add_argument("--return-trace", action="store_true")
-    parser.add_argument("--timeout", type=float, default=60.0)
-    args_ns = parser.parse_args()
-
-    a = Args(
-        base_url=_ensure_str(args_ns.base_url, "--base-url"),
-        api_key=_ensure_str(args_ns.api_key, "--api-key"),
-        inference_url=_ensure_str(args_ns.inference_url, "--inference-url"),
-        inference_api_key=_ensure_str(args_ns.inference_api_key, "--inference-api-key"),
-        model=_ensure_str(args_ns.model, "--model"),
-        episodes=int(args_ns.episodes),
-        start_seed=int(args_ns.start_seed),
-        max_llm_calls=int(args_ns.max_llm_calls),
-        concurrency=max(1, int(args_ns.concurrency)),
-        return_trace=bool(args_ns.return_trace),
-        timeout=float(args_ns.timeout),
-    )
-
-    print(
-        f"[INFO] base={a.base_url} episodes={a.episodes} start_seed={a.start_seed} model={a.model} tp={a.max_llm_calls}"
-    )
-    run_id = f"traces-{int(time.time())}"
-
-    successes = 0
-    failures = 0
-    async with TaskAppClient(a.base_url, api_key=a.api_key, timeout=a.timeout) as client:
-        tasks = [
-            _run_one(client, run_id, seed, a) for seed in range(a.start_seed, a.start_seed + a.episodes)
-        ]
-        for result in await _bounded_gather(a.concurrency, tasks):
-            if isinstance(result, dict):
-                successes += 1
-                print(f"[OK] seed={result['seed']} return={result.get('episode_returns')}")
-            else:
-                failures += 1
-                print(f"[ERR] seed result not dict: {result}", file=sys.stderr)
-
-    print(f"[DONE] successes={successes} failures={failures}")
-
-
-if __name__ == "__main__":
-    try:
-        asyncio.run(main())
-    except KeyboardInterrupt:
-        print("Interrupted", file=sys.stderr)
-
-

examples/swe/__init__.py

@@ -1,12 +0,0 @@
-"""SWE task app examples package."""
-
-from importlib import resources as _resources
-
-__all__ = ["path_for"]
-
-
-def path_for(package: str, resource: str) -> str:
-    """Return path for packaged SWE example resources."""
-
-    with _resources.as_file(_resources.files(f"examples.swe.{package}") / resource) as path:
-        return str(path)

examples/swe/task_app/README.md

@@ -1,105 +0,0 @@
-# mini-SWE Task App
-
-This directory contains an example task app that exposes the
-[mini-swe-agent](https://github.com/SWE-agent/mini-swe-agent) workflow through
-the Synth AI task app interface. The goal is to provide a Crafter-like workflow
-for SWE tasks: you can serve the task app locally, point RL training at it,
-collect rollouts with tracing, and run vendor inference via the standard proxy
-endpoints.
-
-> **Status:** The implementation focuses on a minimal, hackable integration.
-> It supports local/docker environments, step-wise command execution, tracing
-> hooks, and rollouts. By default it streams SWE-Bench Verified tasks from
-> Hugging Face; you can point the loader at your own dataset (or the bundled
-> sample JSONL) via environment variables (see the docs at the end of this
-> file).
-
-## Layout
-
-- `grpo_swe_mini.py` – main task-app configuration (dataset, rollout executor,
-  tracing, Modal metadata, registration).
-- `grpo_swe_mini_task_app.py` – backwards-compatible FastAPI wrapper that
-  allows running the module directly (mirrors `grpo_crafter_task_app.py`).
-- `hosted/envs/mini_swe` – environment/policy adapters that wrap `mini-swe-agent`
-  inside a hosted FastAPI service.
-- `data/sample_instances.json` – optional curated subset for quick smoke tests
-  (no longer the default dataset).
-
-## Using the task app
-
-```
-uvx synth-ai serve swe-mini --port 8020
-```
-
-### Recommended: non-interactive serve + .env
-
-```bash
-uvx synth-ai serve swe-mini \
-  --port 8020 \
-  --env-file .env \
-  --trace traces/v3 \
-  --trace-db traces/v3/synth_ai.db
-```
-
-This avoids interactive prompts (useful for CI) and loads `ENVIRONMENT_API_KEY`, `OPENAI_API_KEY`, etc. from `.env`.
-
-### Configure dataset and execution
-
-Set `SWE_MINI_DATASET` to control what tasks the environment loads (defaults to
-`hf://princeton-nlp/SWE-Bench_Verified:test`):
-
-- `file://path/to/tasks.jsonl` – each line should contain an object with
-  `instance_id`, `problem_statement`, and optional docker metadata
-  (`image_name`, `repo`, …).
-- `hf://namespace/dataset:split` – lazily stream from Hugging Face (requires
-  `datasets` and network).
-  For quick local smoke tests you can point at
-  `file://$REPO/examples/swe/task_app/data/sample_instances.json`.
-
-Execution is handled by mini-swe's environment classes. Configure execution via
-`SWE_MINI_ENVIRONMENT_CLASS` (`local`, `docker`, `singularity`, …) and pass
-additional keyword arguments with `SWE_MINI_ENVIRONMENT_KWARGS` (JSON).
-
-### Tracing & SFT
-
-Tracing works the same as Crafter; pass `--trace` / `--trace-db` to the CLI or
-set `TASKAPP_TRACING_ENABLED=1`. The task app writes JSONL snippets for SFT and
-records decision traces in the configured SQLite/Postgres database.
-
-## Next steps
-
-- `docs/examples/swe/mini_swe_task_app.md` – end-to-end walkthrough
-- `examples/swe/task_app/grpo_swe_mini.py` – main entrypoint
-- `examples/swe/task_app/hosted` – shared host scaffolding for the Mini-SWE task app
-
-Pull requests welcome – especially for better dataset loaders, richer metrics,
-and robust docker support.
-
-### Example rollout configs
-
-- OpenAI gpt-4o-mini (works out-of-the-box):
-
-```json
-{
-  "run_id": "example-$(date +%s)",
-  "policy": {
-    "policy_name": "swe-mini-react",
-    "config": {
-      "model": "gpt-4o-mini",
-      "inference_url": "https://api.openai.com",
-      "temperature": 0.0,
-      "max_completion_tokens": 256,
-      "use_tools": false,
-      "response_format": { "type": "text" },
-      "system_template": "You are participating in a software engineering evaluation. Provide exactly one bash command enclosed in a single ```bash``` block. No THOUGHT. No extra text. If unsure, output ```bash\necho NOOP\n```.",
-      "instance_template": "{{problem_statement}}\n\n{{instructions}}",
-      "action_template": "{{ output.stdout }}"
-    }
-  },
-  "env": { "env_name": "swe-mini" },
-  "ops": ["agent","env","agent","env","agent","env"],
-  "record": {"trajectories": true, "return_trace": true, "trace_format": "compact"}
-}
-```
-
-- OpenAI gpt-5-mini (experimental): remove reasoning flags and constrain output. If responses are empty, retry without `stop` and consider switching to `gpt-4o-mini`.

examples/swe/task_app/__init__.py

@@ -1,2 +0,0 @@
-"""Task app configuration for the mini-SWE integration."""
-