synth-ai 0.2.9.dev4__py3-none-any.whl → 0.2.9.dev6__py3-none-any.whl

examples/warming_up_to_rl/run_fft_and_save.py

@@ -6,12 +6,13 @@ import json
 import os
 import sys
 import time
+import tomllib
 from pathlib import Path
-from typing import Any, Dict, Tuple, List
+from typing import Any
 
-import tomllib
-import re
 import requests
+from dotenv import load_dotenv
+from synth_ai.config.base_url import PROD_BASE_URL_DEFAULT
 
 
 def mask(val: str) -> str:
@@ -20,7 +21,9 @@ def mask(val: str) -> str:
     return f"{val[:6]}…{val[-4:]}" if len(val) >= 10 else "****"
 
 
-def post_multipart(base: str, api_key: str, path: str, file_field: str, filepath: Path) -> Dict[str, Any]:
+def post_multipart(
+    base: str, api_key: str, path: str, file_field: str, filepath: Path
+) -> dict[str, Any]:
     """Upload a file, trying backend-specific endpoints with fallbacks.
 
     Priority:
@@ -33,9 +36,9 @@ def post_multipart(base: str, api_key: str, path: str, file_field: str, filepath
 
     endpoints = [
         f"{base.rstrip('/')}/{path.lstrip('/')}",  # e.g., /learning/files
-        f"{base.rstrip('/')}/files",               # OpenAI-style
+        f"{base.rstrip('/')}/files",  # OpenAI-style
     ]
-    last_err: Dict[str, Any] | None = None
+    last_err: dict[str, Any] | None = None
     for ep in endpoints:
         try:
             r = requests.post(ep, headers=headers, files=files, data=data, timeout=300)
@@ -67,7 +70,7 @@ def post_multipart(base: str, api_key: str, path: str, file_field: str, filepath
     return last_err or {"error": True, "detail": "upload_failed_all_endpoints"}
 
 
-def post_json(base: str, api_key: str, path: str, body: Dict[str, Any]) -> Dict[str, Any]:
+def post_json(base: str, api_key: str, path: str, body: dict[str, Any]) -> dict[str, Any]:
     url = f"{base.rstrip('/')}/{path.lstrip('/')}"
     headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
     r = requests.post(url, headers=headers, data=json.dumps(body), timeout=120)
@@ -77,7 +80,7 @@ def post_json(base: str, api_key: str, path: str, body: Dict[str, Any]) -> Dict[
         return {"status": r.status_code, "text": r.text[:400]}
 
 
-def get_json(base: str, api_key: str, path: str) -> Dict[str, Any]:
+def get_json(base: str, api_key: str, path: str) -> dict[str, Any]:
     url = f"{base.rstrip('/')}/{path.lstrip('/')}"
     headers = {"Authorization": f"Bearer {api_key}"}
     r = requests.get(url, headers=headers, timeout=30)
@@ -87,17 +90,94 @@ def get_json(base: str, api_key: str, path: str) -> Dict[str, Any]:
         return {"status": r.status_code, "text": r.text[:400]}
 
 
+def _find_fft_configs() -> list[Path]:
+    """Find FFT TOML configs in standard locations."""
+    candidates: list[Path] = []
+
+    # Check current directory configs/
+    cwd = Path.cwd()
+    configs_dir = cwd / "configs"
+    if configs_dir.is_dir():
+        for f in configs_dir.glob("*.toml"):
+            # Look for FFT configs (check if they have [algorithm] method = "supervised_finetune")
+            try:
+                content = f.read_text()
+                if "supervised_finetune" in content or "fft" in content.lower():
+                    candidates.append(f)
+            except Exception:
+                pass
+
+    # Also check for any .toml files in current directory
+    for f in cwd.glob("*.toml"):
+        if f not in candidates:
+            try:
+                content = f.read_text()
+                if "supervised_finetune" in content or "fft" in content.lower():
+                    candidates.append(f)
+            except Exception:
+                pass
+
+    return sorted(candidates)
+
+
 def main() -> None:
+    # Load .env file from current directory first if it exists
+    default_env = Path.cwd() / ".env"
+    if default_env.exists():
+        load_dotenv(default_env, override=False)
+
     parser = argparse.ArgumentParser(description="Submit FFT job and save resulting model id")
-    parser.add_argument("--backend", default=os.getenv("BACKEND_BASE_URL", "http://localhost:8000/api"))
-    parser.add_argument("--toml", required=True, help="Path to FFT TOML config")
+    parser.add_argument(
+        "--backend", default=os.getenv("BACKEND_BASE_URL", f"{PROD_BASE_URL_DEFAULT}/api")
+    )
+    parser.add_argument("--toml", required=False, help="Path to FFT TOML config")
     parser.add_argument("--data", default="", help="Override dataset JSONL path")
     parser.add_argument("--poll-seconds", type=int, default=1800)
-    parser.add_argument("--env-file", default="", help="Optional path to .env file with SYNTH_API_KEY")
+    parser.add_argument(
+        "--env-file", default="", help="Optional path to .env file with SYNTH_API_KEY"
+    )
     args = parser.parse_args()
 
-    config_path = Path(args.toml).expanduser().resolve()
-    if not config_path.exists():
+    # Also load from explicit --env-file if provided
+    if args.env_file:
+        env_path = Path(args.env_file).expanduser()
+        if not env_path.exists():
+            print(f"[WARN] Env file not found: {env_path}")
+        else:
+            load_dotenv(env_path, override=False)
+
+    # Auto-discover TOML config if not specified
+    config_path: Path | None = None
+    if args.toml:
+        config_path = Path(args.toml).expanduser().resolve()
+    else:
+        configs = _find_fft_configs()
+        if not configs:
+            print(
+                "No FFT config files found. Please specify --toml or create a config in configs/",
+                file=sys.stderr,
+            )
+            sys.exit(2)
+        elif len(configs) == 1:
+            config_path = configs[0]
+            print(f"Using FFT config: {config_path}")
+        else:
+            print("\nFound multiple FFT configs:")
+            for idx, cfg in enumerate(configs, 1):
+                print(f"  [{idx}] {cfg}")
+            choice = input(f"Select config [1-{len(configs)}]: ").strip()
+            try:
+                selected_idx = int(choice) - 1
+                if 0 <= selected_idx < len(configs):
+                    config_path = configs[selected_idx]
+                else:
+                    print("Invalid selection", file=sys.stderr)
+                    sys.exit(2)
+            except ValueError:
+                print("Invalid input", file=sys.stderr)
+                sys.exit(2)
+
+    if not config_path or not config_path.exists():
         print(f"Config not found: {config_path}", file=sys.stderr)
         sys.exit(2)
     with config_path.open("rb") as fh:
@@ -107,7 +187,9 @@ def main() -> None:
     compute_cfg = cfg.get("compute", {}) if isinstance(cfg.get("compute"), dict) else {}
     data_cfg_full = cfg.get("data", {}) if isinstance(cfg.get("data"), dict) else {}
     topo_cfg = (data_cfg_full or {}).get("topology", {}) if isinstance(data_cfg_full, dict) else {}
-    validation_local_path = (data_cfg_full or {}).get("validation_path") if isinstance(data_cfg_full, dict) else None
+    validation_local_path = (
+        (data_cfg_full or {}).get("validation_path") if isinstance(data_cfg_full, dict) else None
+    )
     train_cfg = cfg.get("training", {}) if isinstance(cfg.get("training"), dict) else {}
     hp_cfg = cfg.get("hyperparameters", {}) if isinstance(cfg.get("hyperparameters"), dict) else {}
 
@@ -119,7 +201,10 @@ def main() -> None:
     if isinstance(data_path, str) and data_path.strip():
         p = Path(data_path).expanduser()
         if not p.is_absolute():
-            p = (config_path.parent / p).resolve()
+            # Try relative to cwd first, then relative to config directory
+            cwd_relative = Path.cwd() / p
+            config_relative = config_path.parent / p
+            p = cwd_relative.resolve() if cwd_relative.exists() else config_relative.resolve()
         data_file = p
     if data_file is None:
         print("Missing dataset path in --data or [job].data", file=sys.stderr)
@@ -129,38 +214,11 @@ def main() -> None:
         sys.exit(2)
 
     synth_key = (os.getenv("SYNTH_API_KEY") or "").strip()
-    # Fallback: try to load from .env if not present in environment
     if not synth_key:
-        candidate_env: Path | None = None
-        if isinstance(args.env_file, str) and args.env_file.strip():
-            candidate_env = Path(args.env_file).expanduser().resolve()
-        else:
-            # Prefer .env next to the TOML config
-            candidate_env = (config_path.parent / ".env").resolve()
-        if candidate_env and candidate_env.exists():
-            try:
-                env_text = candidate_env.read_text(encoding="utf-8", errors="ignore")
-                # Match lines like: SYNTH_API_KEY=..., or export SYNTH_API_KEY=...
-                key_val: str | None = None
-                for line in env_text.splitlines():
-                    m = re.match(r"^\s*(?:export\s+)?SYNTH_API_KEY\s*=\s*(.*)$", line)
-                    if m:
-                        raw = m.group(1).strip()
-                        # Trim surrounding quotes if present
-                        if (raw.startswith('"') and raw.endswith('"')) or (raw.startswith("'") and raw.endswith("'")):
-                            raw = raw[1:-1]
-                        key_val = raw.strip()
-                        break
-                if key_val:
-                    synth_key = key_val
-                    os.environ["SYNTH_API_KEY"] = synth_key
-                    print(f"[INFO] Loaded SYNTH_API_KEY from {candidate_env}")
-            except Exception as _e:
-                # Ignore and fall through to error below
-                pass
-    if not synth_key:
-        print("Missing SYNTH_API_KEY (set in env or provide --env-file pointing to .env)", file=sys.stderr)
-        sys.exit(2)
+        synth_key = input("Please enter your Synth API key:\n> ").strip()
+        if not synth_key:
+            print("Synth API key is required", file=sys.stderr)
+            sys.exit(2)
 
     backend = args.backend.rstrip("/")
     print(f"[INFO] Using backend={backend} key_fp={mask(synth_key)} data={data_file}")
@@ -180,7 +238,10 @@ def main() -> None:
         err_status = (upf or {}).get("status")
         err_body = (upf or {}).get("body") or (upf or {}).get("text")
         err_ep = (upf or {}).get("endpoint")
-        print(f"Upload failed (status={err_status} endpoint={err_ep}) body={str(err_body)[:200]}", file=sys.stderr)
+        print(
+            f"Upload failed (status={err_status} endpoint={err_ep}) body={str(err_body)[:200]}",
+            file=sys.stderr,
+        )
         sys.exit(4)
 
     # Optionally upload validation file
@@ -203,10 +264,12 @@ def main() -> None:
                 err_status = (upv or {}).get("status")
                 err_body = (upv or {}).get("body") or (upv or {}).get("text")
                 err_ep = (upv or {}).get("endpoint")
-                print(f"[WARN] Validation upload failed (status={err_status} endpoint={err_ep}) body={str(err_body)[:180]} — continuing without validation")
+                print(
+                    f"[WARN] Validation upload failed (status={err_status} endpoint={err_ep}) body={str(err_body)[:180]} — continuing without validation"
+                )
 
     # 2) Build job payload
-    hp_block: Dict[str, Any] = {
+    hp_block: dict[str, Any] = {
         "n_epochs": int(hp_cfg.get("n_epochs") or 1),
     }
     # Optional extras if present
@@ -227,7 +290,7 @@ def main() -> None:
     if parallel:
         hp_block["parallelism"] = parallel
 
-    compute_block: Dict[str, Any] = {}
+    compute_block: dict[str, Any] = {}
     for k in ("gpu_type", "gpu_count", "nodes"):
         if k in compute_cfg:
             compute_block[k] = compute_cfg[k]
@@ -238,18 +301,24 @@ def main() -> None:
         "training": {k: v for k, v in train_cfg.items() if k in ("mode", "use_qlora")},
     }
     # If TOML includes a [training.validation] block, forward relevant knobs into hyperparameters
-    validation_cfg = train_cfg.get("validation") if isinstance(train_cfg.get("validation"), dict) else None
+    validation_cfg = (
+        train_cfg.get("validation") if isinstance(train_cfg.get("validation"), dict) else None
+    )
     if isinstance(validation_cfg, dict):
         # Enable evaluation and map keys as-is; backend trainer maps metric_for_best_model 'val.loss'→'eval_loss'
-        hp_block.update({
-            "evaluation_strategy": validation_cfg.get("evaluation_strategy", "steps"),
-            "eval_steps": int(validation_cfg.get("eval_steps", 0) or 0),
-            "save_best_model_at_end": bool(validation_cfg.get("save_best_model_at_end", True)),
-            "metric_for_best_model": validation_cfg.get("metric_for_best_model", "val.loss"),
-            "greater_is_better": bool(validation_cfg.get("greater_is_better", False)),
-        })
+        hp_block.update(
+            {
+                "evaluation_strategy": validation_cfg.get("evaluation_strategy", "steps"),
+                "eval_steps": int(validation_cfg.get("eval_steps", 0) or 0),
+                "save_best_model_at_end": bool(validation_cfg.get("save_best_model_at_end", True)),
+                "metric_for_best_model": validation_cfg.get("metric_for_best_model", "val.loss"),
+                "greater_is_better": bool(validation_cfg.get("greater_is_better", False)),
+            }
+        )
         # Also surface validation enable flag into effective_config for visibility (optional)
-        effective.setdefault("training", {})["validation"] = {"enabled": bool(validation_cfg.get("enabled", True))}
+        effective.setdefault("training", {})["validation"] = {
+            "enabled": bool(validation_cfg.get("enabled", True))
+        }
 
     body = {
         "model": model,
@@ -289,7 +358,9 @@ def main() -> None:
             break
         # Warn if stuck queued for >10 minutes
         if status == "queued" and (time.time() - queued_since) > 600:
-            print("[WARN] Job has remained queued for >10 minutes. Backend may be capacity constrained.")
+            print(
+                "[WARN] Job has remained queued for >10 minutes. Backend may be capacity constrained."
+            )
             queued_since = time.time()
         time.sleep(5)
 

examples/warming_up_to_rl/run_local_rollout.py

@@ -7,14 +7,12 @@ import argparse
 import asyncio
 import json
 import os
+import sys
 from pathlib import Path
 from typing import Any
 
-import sys
-
 import httpx
 from dotenv import load_dotenv
-
 from synth_ai.task import (
     RolloutEnvSpec,
     RolloutPolicySpec,
@@ -46,17 +44,21 @@ def build_rollout_request(
     )
     return RolloutRequest(
         run_id=run_id,
-        env=RolloutEnvSpec(env_name='crafter', seed=seed, config={}),
-        policy=RolloutPolicySpec(policy_name='crafter-react', config=policy_config),
+        env=RolloutEnvSpec(env_name="crafter", seed=seed, config={}),
+        policy=RolloutPolicySpec(policy_name="crafter-react", config=policy_config),
         ops=ops,
         record=record_cfg,
-        on_done='reset',
+        on_done="reset",
         safety=RolloutSafetyConfig(),
     )
 
 
 def summarise_response(data: Any) -> dict[str, Any]:
-    metrics = data.metrics.model_dump() if hasattr(data.metrics, "model_dump") else data.get("metrics", {})
+    metrics = (
+        data.metrics.model_dump()
+        if hasattr(data.metrics, "model_dump")
+        else data.get("metrics", {})
+    )
     error = None
     rollout_status = None
     try:
@@ -86,16 +88,42 @@ async def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--base-url", default="http://localhost:8001", help="Task app base URL")
     parser.add_argument("--api-key", help="Environment API key (or set via --env-file)")
-    parser.add_argument('--seed', type=int, default=42, help='Env seed to rollout')
-    parser.add_argument('--run-id', default='local-demo', help='Run identifier')
-    parser.add_argument('--model', default='gpt-4o-mini', help='Model identifier for the Crafter policy (OpenAI-compatible)')
-    parser.add_argument('--inference-url', default='https://api.openai.com', help='Inference base URL used by the policy (e.g., https://api.openai.com)')
-    parser.add_argument('--env-file', type=str, default=None, help='Path to .env file with API keys')
-    parser.add_argument('--ops', default=None, help='Comma-separated rollout ops (advanced override)')
-    parser.add_argument('--max-llm-calls', type=int, default=1, help='Number of policy inference calls when --ops not provided')
-    parser.add_argument('--max-policy-tokens', type=int, default=None, help='Optional per-call token limit forwarded to the policy config')
-    parser.add_argument('--timeout', type=float, default=600.0, help='HTTP timeout (seconds) for task app requests')
-    parser.add_argument('--verbose', action='store_true', help='Print resolved configuration and headers')
+    parser.add_argument("--seed", type=int, default=42, help="Env seed to rollout")
+    parser.add_argument("--run-id", default="local-demo", help="Run identifier")
+    parser.add_argument(
+        "--model",
+        default="gpt-4o-mini",
+        help="Model identifier for the Crafter policy (OpenAI-compatible)",
+    )
+    parser.add_argument(
+        "--inference-url",
+        default="https://api.openai.com",
+        help="Inference base URL used by the policy (e.g., https://api.openai.com)",
+    )
+    parser.add_argument(
+        "--env-file", type=str, default=None, help="Path to .env file with API keys"
+    )
+    parser.add_argument(
+        "--ops", default=None, help="Comma-separated rollout ops (advanced override)"
+    )
+    parser.add_argument(
+        "--max-llm-calls",
+        type=int,
+        default=1,
+        help="Number of policy inference calls when --ops not provided",
+    )
+    parser.add_argument(
+        "--max-policy-tokens",
+        type=int,
+        default=None,
+        help="Optional per-call token limit forwarded to the policy config",
+    )
+    parser.add_argument(
+        "--timeout", type=float, default=600.0, help="HTTP timeout (seconds) for task app requests"
+    )
+    parser.add_argument(
+        "--verbose", action="store_true", help="Print resolved configuration and headers"
+    )
     args = parser.parse_args()
 
     if args.env_file:
@@ -117,12 +145,13 @@ async def main() -> None:
             os.environ["OPENAI_API_KEY"] = synth_key
 
     if args.verbose:
+
         def _mask(val: str | None) -> str:
             if not val:
-                return '<unset>'
+                return "<unset>"
             return f"{val[:6]}…{val[-4:]} (len={len(val)})"
 
-        print('Resolved configuration:')
+        print("Resolved configuration:")
         print(f"  Task app base URL  : {args.base_url}")
         print(f"  Inference base URL : {args.inference_url}")
         print(f"  Task app API key   : {_mask(api_key)}")
@@ -130,21 +159,23 @@ async def main() -> None:
         print(f"  HTTP timeout       : {args.timeout:.1f}s")
 
     if args.ops:
-        ops = [op.strip() for op in args.ops.split(',') if op.strip()]
+        ops = [op.strip() for op in args.ops.split(",") if op.strip()]
         if not ops:
-            raise ValueError('Ops must contain at least one entry')
+            raise ValueError("Ops must contain at least one entry")
     else:
         llm_calls = max(args.max_llm_calls, 1)
         if llm_calls > 20:
-            print('[WARN] --max-llm-calls capped at 20 to avoid excessive episodes; use --ops for manual control.')
+            print(
+                "[WARN] --max-llm-calls capped at 20 to avoid excessive episodes; use --ops for manual control."
+            )
             llm_calls = 20
         ops = []
         for _ in range(llm_calls):
-            ops.extend(['agent', 'env'])
+            ops.extend(["agent", "env"])
 
     async with TaskAppClient(args.base_url, api_key=api_key, timeout=args.timeout) as client:
         try:
-            print(f'Fetching task_info for seed {args.seed}…')
+            print(f"Fetching task_info for seed {args.seed}…")
             task_info = await client.task_info(seeds=[args.seed])
             info_payload = task_info[0] if isinstance(task_info, list) else task_info
             print(json.dumps(info_payload.model_dump(), indent=2)[:600])
@@ -158,29 +189,47 @@ async def main() -> None:
                 extra_headers=extra_headers,
             )
             if args.max_policy_tokens is not None:
-                request.policy.config.update({
-                    'max_completion_tokens': args.max_policy_tokens,
-                    'max_tokens': args.max_policy_tokens,
-                })
+                request.policy.config.update(
+                    {
+                        "max_completion_tokens": args.max_policy_tokens,
+                        "max_tokens": args.max_policy_tokens,
+                    }
+                )
             if args.verbose:
-                print(f'Ops: {ops}')
-                print(f'Request headers: {request.policy.config.get("extra_headers", {})}')
-            print('Requesting rollout…')
+                print(f"Ops: {ops}")
+                print(f"Request headers: {request.policy.config.get('extra_headers', {})}")
+            print("Requesting rollout…")
             response = await client.rollout(request)
             summary = summarise_response(response)
             print(json.dumps(summary, indent=2))
-            print(f'Ops executed: {ops}')
-            print('Tip: use --max-llm-calls N for agent/env pairs or --ops for manual control.')
+            print(f"Ops executed: {ops}")
+            print("Tip: use --max-llm-calls N for agent/env pairs or --ops for manual control.")
         except httpx.HTTPStatusError as exc:
-            detail = exc.response.json() if exc.response.headers.get('content-type', '').startswith('application/json') else exc.response.text
-            print(f'HTTP error {exc.response.status_code}: {detail}', file=sys.stderr)
+            detail = (
+                exc.response.json()
+                if exc.response.headers.get("content-type", "").startswith("application/json")
+                else exc.response.text
+            )
+            print(f"HTTP error {exc.response.status_code}: {detail}", file=sys.stderr)
             if exc.response.status_code in (401, 503):
-                print('Hint: ensure the task app was started with ENVIRONMENT_API_KEY set and pass the same key via --api-key.', file=sys.stderr)
+                print(
+                    "Hint: ensure the task app was started with ENVIRONMENT_API_KEY set and pass the same key via --api-key.",
+                    file=sys.stderr,
+                )
             if exc.response.status_code == 500 and args.model in str(detail):
-                print('Hint: supply --model/--inference-url (and set OPENAI_API_KEY or GROQ_API_KEY) so the policy can route inference.', file=sys.stderr)
-                print('Hint: the inference URL should be the base (e.g., https://api.openai.com); the task app appends /v1/chat/completions.', file=sys.stderr)
+                print(
+                    "Hint: supply --model/--inference-url (and set OPENAI_API_KEY or GROQ_API_KEY) so the policy can route inference.",
+                    file=sys.stderr,
+                )
+                print(
+                    "Hint: the inference URL should be the base (e.g., https://api.openai.com); the task app appends /v1/chat/completions.",
+                    file=sys.stderr,
+                )
                 if args.max_policy_tokens is not None:
-                    print(f'Hint: --max-policy-tokens={args.max_policy_tokens} is forwarded to the policy config as max_completion_tokens.', file=sys.stderr)
+                    print(
+                        f"Hint: --max-policy-tokens={args.max_policy_tokens} is forwarded to the policy config as max_completion_tokens.",
+                        file=sys.stderr,
+                    )
             raise