synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/utils/uvicorn.py

@@ -0,0 +1,113 @@
+import importlib.util as import_util
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from synth_ai.task_app_cfgs import LocalTaskAppConfig
+from synth_ai.utils.env import resolve_env_var
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+START_DIV = f"{'-' * 30} Uvicorn start {'-' * 30}"
+END_DIV = f"{'-' * 31} Uvicorn end {'-' * 31}"
+
+_ASGI_FACTORY_NAMES = (
+    "fastapi_app",
+    "create_app",
+    "build_app",
+    "configure_app",
+    "get_app",
+    "app_factory",
+)
+
+
+def _coerce_asgi_app(candidate: Any) -> Any | None:
+    if candidate is None:
+        return None
+    if callable(candidate):
+        return candidate
+    return None
+
+
+def deploy_uvicorn_app(cfg: LocalTaskAppConfig) -> None:
+    task_app_path = cfg.task_app_path.resolve()
+
+    env_key = resolve_env_var("ENVIRONMENT_API_KEY")
+    if not env_key:
+        raise RuntimeError("ENVIRONMENT_API_KEY is required to serve locally.")
+
+    if cfg.trace:
+        os.environ["TASKAPP_TRACING_ENABLED"] = "1"
+    else:
+        os.environ.pop("TASKAPP_TRACING_ENABLED", None)
+
+    task_app_dir = task_app_path.parent.resolve()
+    candidates: list[Path] = [task_app_dir]
+    if (task_app_dir / "__init__.py").exists():
+        candidates.append(task_app_dir.parent.resolve())
+    candidates.append(REPO_ROOT)
+
+    unique: list[str] = []
+    for candidate in candidates:
+        candidate_str = str(candidate)
+        if candidate_str and candidate_str not in unique:
+            unique.append(candidate_str)
+
+    existing = os.environ.get("PYTHONPATH")
+    if existing:
+        for segment in existing.split(os.pathsep):
+            if segment and segment not in unique:
+                unique.append(segment)
+
+    os.environ["PYTHONPATH"] = os.pathsep.join(unique)
+    for entry in reversed(unique):
+        if entry and entry not in sys.path:
+            sys.path.insert(0, entry)
+
+    module_name = f"_synth_local_task_app_{task_app_path.stem}"
+    spec = import_util.spec_from_file_location(module_name, str(task_app_path))
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Unable to load task app at {task_app_path}")
+    module = import_util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    try:
+        spec.loader.exec_module(module)  # type: ignore[call-arg]
+    except Exception as exc:
+        raise RuntimeError(f"Failed to import task app: {exc}") from exc
+
+    app = _coerce_asgi_app(getattr(module, "app", None))
+    if app is None:
+        for name in _ASGI_FACTORY_NAMES:
+            factory = getattr(module, name, None)
+            if callable(factory):
+                produced = factory()
+                coerced = _coerce_asgi_app(produced)
+                if coerced is not None:
+                    app = coerced
+                    break
+    if app is None:
+        raise RuntimeError("Task app must expose an ASGI application via `app = FastAPI(...)` or a callable factory.")
+
+    host = cfg.host
+    port = cfg.port
+    preview_host = "127.0.0.1" if host in {"0.0.0.0", "::"} else host
+    print(f"[uvicorn] Serving task app at http://{preview_host}:{port}")
+
+
+# Deploy
+    try:
+        import uvicorn  # type: ignore
+    except ImportError as exc:
+        raise RuntimeError(
+            "uvicorn is required to serve task apps locally. Install it with `pip install uvicorn`."
+        ) from exc
+
+    try:
+        print(START_DIV)
+        uvicorn.run(app, host=host, port=port, reload=False, log_level="info")
+    except KeyboardInterrupt:
+        print("\n[uvicorn] Stopped by user.")
+    except Exception as exc:
+        raise RuntimeError(f"uvicorn runtime failed: {exc}") from exc
+    finally:
+        print(END_DIV)

{synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.17
+Version: 0.2.19
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT
@@ -23,8 +23,8 @@ Requires-Dist: rich>=13.9.0
 Requires-Dist: openai>=1.99.0
 Requires-Dist: anthropic>=0.42.0
 Requires-Dist: langfuse<3.0.0,>=2.53.9
-Requires-Dist: opentelemetry-api<1.27.0,>=1.26.0
-Requires-Dist: opentelemetry-sdk<1.27.0,>=1.26.0
+Requires-Dist: opentelemetry-api>=1.26.0
+Requires-Dist: opentelemetry-sdk>=1.26.0
 Requires-Dist: diskcache>=5.6.3
 Requires-Dist: groq>=0.30.0
 Requires-Dist: google-genai>=1.26.0
@@ -53,9 +53,10 @@ Requires-Dist: aiohttp>=3.8.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: datasets>=4.0.0
 Requires-Dist: transformers>=4.56.1
-Requires-Dist: modal==1.1.4
+Requires-Dist: modal<2.0.0,>=1.1.4
 Requires-Dist: pyboy>=2.6.0
 Requires-Dist: setuptools>=80.9.0
+Requires-Dist: libsql-experimental>=0.0.55
 Provides-Extra: dev
 Requires-Dist: build>=1.2.2.post1; extra == "dev"
 Requires-Dist: twine>=4.0.0; extra == "dev"
@@ -118,6 +119,7 @@ uvx synth-ai setup
 uvx synth-ai demo
 uvx synth-ai deploy
 uvx synth-ai run
+uvx synth-ai baseline  # For coding agents: get baseline scores
 ```
 
 > Full quickstart: [https://docs.usesynth.ai/sdk/get-started](https://docs.usesynth.ai/sdk/get-started)
@@ -158,6 +160,102 @@ Synth-AI ships with a built-in RL example: training **Qwen3-0.6B** on math reaso
 
 ---
 
+## 🤖 For Coding Agents: Get Started with Baselines
+
+**Baselines** are the fastest way for coding agents to evaluate changes and measure improvement on Synth tasks.
+
+### Why Use Baselines?
+
+Baselines provide a **self-contained evaluation system** that:
+- ✅ **No infrastructure required** — runs locally, no deployed task app needed
+- ✅ **Quick feedback loop** — get task-by-task results in seconds
+- ✅ **Compare changes** — establish a baseline score before making modifications
+- ✅ **Auto-discoverable** — finds baseline files automatically in your codebase
+
+### Quick Start for Coding Agents
+
+```bash
+# 1. List available baselines
+uvx synth-ai baseline list
+
+# 2. Run a quick 3-task baseline to get started
+uvx synth-ai baseline banking77 --split train --seeds 0,1,2
+
+# 3. Get your baseline score (full train split)
+uvx synth-ai baseline banking77 --split train
+
+# 4. Make your changes to the code...
+
+# 5. Re-run to compare performance
+uvx synth-ai baseline banking77 --split train --output results_after.json
+```
+
+### Available Baselines
+
+```bash
+# Filter by task type
+uvx synth-ai baseline list --tag rl          # RL tasks
+uvx synth-ai baseline list --tag nlp         # NLP tasks
+uvx synth-ai baseline list --tag vision      # Vision tasks
+
+# Run specific baselines
+uvx synth-ai baseline warming_up_to_rl       # Crafter survival game
+uvx synth-ai baseline pokemon_vl             # Pokemon Red (vision)
+uvx synth-ai baseline gepa                   # Banking77 classification
+```
+
+### Baseline Results
+
+Each baseline run provides:
+- **Task-by-task results** — see exactly which seeds succeed/fail
+- **Aggregate metrics** — success rate, mean/std rewards, total tasks
+- **Serializable output** — save to JSON with `--output results.json`
+- **Model comparison** — test different models with `--model`
+
+Example output:
+```
+============================================================
+Baseline Evaluation: Banking77 Intent Classification
+============================================================
+Split(s): train
+Tasks: 10
+Success: 8/10
+Execution time: 12.34s
+
+Aggregate Metrics:
+  mean_outcome_reward: 0.8000
+  success_rate: 0.8000
+  total_tasks: 10
+```
+
+### Creating Custom Baselines
+
+Coding agents can create new baseline files to test custom tasks:
+
+```python
+# my_task_baseline.py
+from synth_ai.baseline import BaselineConfig, BaselineTaskRunner, DataSplit, TaskResult
+
+class MyTaskRunner(BaselineTaskRunner):
+    async def run_task(self, seed: int) -> TaskResult:
+        # Your task logic here
+        return TaskResult(...)
+
+my_baseline = BaselineConfig(
+    baseline_id="my_task",
+    name="My Custom Task",
+    description="Evaluate my custom task",
+    task_runner=MyTaskRunner,
+    splits={
+        "train": DataSplit(name="train", seeds=list(range(10))),
+    },
+)
+```
+
+Place this file in `examples/baseline/` or name it `*_baseline.py` for auto-discovery.
+
+---
+
 ## 🔐 SDK → Dashboard Pairing
 
 When you run `uvx synth-ai setup` (or legacy `uvx synth-ai rl_demo setup`):