synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

{synth_ai/task/apps → examples/rl/task_app}/math_single_step.py

@@ -1,22 +1,23 @@
-from __future__ import annotations
-
 """Task app configuration for a single-step math reasoning environment."""
 
+from __future__ import annotations
+
 import contextlib
 import os
 import random
 import re
 import uuid
+from collections.abc import Iterable, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Sequence, cast
+from typing import Any, cast
 
 import httpx
 from datasets import load_dataset
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel, Field
-
-from ..contracts import (
+from synth_ai.task.apps import ModalDeploymentConfig, TaskAppEntry, register_task_app
+from synth_ai.task.contracts import (
     RolloutMetrics,
     RolloutRequest,
     RolloutResponse,
@@ -24,24 +25,25 @@ from ..contracts import (
     RolloutTrajectory,
     TaskInfo,
 )
-from ..datasets import TaskDatasetRegistry, TaskDatasetSpec
-from ..rubrics import Rubric, load_rubric
-from ..server import ProxyConfig, RubricBundle, TaskAppConfig
-from ..errors import http_exception
-from ..tracing_utils import (
+from synth_ai.task.datasets import TaskDatasetRegistry, TaskDatasetSpec
+from synth_ai.task.errors import http_exception
+from synth_ai.task.rubrics import Rubric, load_rubric
+from synth_ai.task.server import ProxyConfig, RubricBundle, TaskAppConfig
+from synth_ai.task.tracing_utils import (
     build_tracer_factory,
     resolve_sft_output_dir,
     resolve_tracing_db_url,
     tracing_env_enabled,
 )
-from ..vendors import normalize_vendor_keys
-from . import ModalDeploymentConfig, TaskAppEntry, register_task_app
+from synth_ai.task.vendors import normalize_vendor_keys
 from synth_ai.tracing_v3.session_tracer import SessionTracer
 
 REPO_ROOT = Path(__file__).resolve().parents[3]
 
-_modal_volume_candidate = Path(os.getenv("MATH_MODAL_DATASET_DIR", "/modal_volumes/math_dataset")).expanduser()
-_modal_volume_root: Optional[Path] = None
+_modal_volume_candidate = Path(
+    os.getenv("MATH_MODAL_DATASET_DIR", "/modal_volumes/math_dataset")
+).expanduser()
+_modal_volume_root: Path | None = None
 try:
     _modal_volume_candidate.mkdir(parents=True, exist_ok=True)
     _modal_volume_root = _modal_volume_candidate
@@ -55,7 +57,9 @@ if _modal_volume_root is not None:
     local_dataset_dir.mkdir(parents=True, exist_ok=True)
     os.environ.setdefault("MATH_DATASET_LOCAL_DIR", str(local_dataset_dir))
 else:
-    hf_cache_path = Path(os.getenv("MATH_DATASET_CACHE_DIR", str(REPO_ROOT / ".cache" / "hf-datasets")) ).expanduser()
+    hf_cache_path = Path(
+        os.getenv("MATH_DATASET_CACHE_DIR", str(REPO_ROOT / ".cache" / "hf-datasets"))
+    ).expanduser()
 
 hf_cache_path.mkdir(parents=True, exist_ok=True)
 os.environ.setdefault("MATH_DATASET_CACHE_DIR", str(hf_cache_path))
@@ -101,7 +105,7 @@ MATH_DATASET_SPEC = TaskDatasetSpec(
 _BOXED_MARKERS: tuple[str, ...] = ("\\boxed", "boxed")
 
 
-def _extract_boxed(text: str) -> Optional[str]:
+def _extract_boxed(text: str) -> str | None:
     if not text:
         return None
     for marker in _BOXED_MARKERS:
@@ -170,9 +174,9 @@ class MathDataset:
         self.name = name
         self.config = config
         self.splits = [split for split in splits if split]
-        self._cache: Dict[str, Any] = {}
+        self._cache: dict[str, Any] = {}
         self._local_dir = os.getenv("MATH_DATASET_LOCAL_DIR")
-        self._hf_token: Optional[str] = None
+        self._hf_token: str | None = None
         for key in HF_TOKEN_ENV_KEYS:
             value = os.getenv(key)
             if value:
@@ -182,7 +186,7 @@ class MathDataset:
                     break
         # No multi-candidate fallback: enforce explicit dataset id
 
-    def _local_file_for_split(self, split: str) -> Optional[Path]:
+    def _local_file_for_split(self, split: str) -> Path | None:
         specific = os.getenv(f"MATH_DATASET_LOCAL_{split.upper()}_FILE")
         if specific:
             path = Path(specific).expanduser()
@@ -203,11 +207,13 @@ class MathDataset:
         if split not in self._cache:
             local_file = self._local_file_for_split(split)
             if local_file is not None:
-                dataset = load_dataset("json", data_files=str(local_file), cache_dir=str(HF_DATASETS_CACHE))
+                dataset = load_dataset(
+                    "json", data_files=str(local_file), cache_dir=str(HF_DATASETS_CACHE)
+                )
                 self._cache[split] = dataset["train"]
             else:
                 try:
-                    load_kwargs: Dict[str, Any] = {"split": split}
+                    load_kwargs: dict[str, Any] = {"split": split}
                     if self.config:
                         load_kwargs["name"] = self.config
                     if self._hf_token:
@@ -221,7 +227,7 @@ class MathDataset:
                             tmp_path = target.with_name(target.name + ".tmp")
                             try:
                                 local_dir.mkdir(parents=True, exist_ok=True)
-                                getattr(ds, "to_json")(str(tmp_path))
+                                ds.to_json(str(tmp_path))
                                 tmp_path.replace(target)
                             except Exception:
                                 with contextlib.suppress(FileNotFoundError):
@@ -235,7 +241,7 @@ class MathDataset:
                     raise RuntimeError(" ".join(hints)) from exc
         return self._cache[split]
 
-    def sample(self, *, split: str, index: Optional[int] = None) -> Dict[str, Any]:
+    def sample(self, *, split: str, index: int | None = None) -> dict[str, Any]:
         dataset = self._load_split(split)
         if len(dataset) == 0:
             raise RuntimeError(f"Dataset split '{split}' is empty")
@@ -301,9 +307,7 @@ class MathDataset:
             except Exception as exc:
                 errors.append(f"{split}: {exc}")
         if errors:
-            raise RuntimeError(
-                "Dataset preparation failed:\n" + "\n".join(errors)
-            )
+            raise RuntimeError("Dataset preparation failed:\n" + "\n".join(errors))
 
 
 @dataclass
@@ -322,9 +326,9 @@ class MathEnvironmentManager:
 
     def __init__(self, dataset: MathDataset) -> None:
         self.dataset = dataset
-        self._states: Dict[str, MathEnvState] = {}
+        self._states: dict[str, MathEnvState] = {}
 
-    def create(self, *, split: str, index: Optional[int], seed: Optional[int]) -> MathEnvState:
+    def create(self, *, split: str, index: int | None, seed: int | None) -> MathEnvState:
         if index is None and seed is not None:
             index = seed
         sample = self.dataset.sample(split=split, index=index)
@@ -350,11 +354,11 @@ class MathEnvironmentManager:
 
 
 class InitializePayload(BaseModel):
-    seed: Optional[int] = None
-    config: Dict[str, Any] = Field(default_factory=dict)
+    seed: int | None = None
+    config: dict[str, Any] = Field(default_factory=dict)
 
 
-def _observation_from_state(state: MathEnvState) -> Dict[str, Any]:
+def _observation_from_state(state: MathEnvState) -> dict[str, Any]:
     return {
         "problem": state.problem,
         "split": state.split,
@@ -362,7 +366,9 @@ def _observation_from_state(state: MathEnvState) -> Dict[str, Any]:
     }
 
 
-def _score_submission(state: MathEnvState, tool_calls: Sequence[Mapping[str, Any]]) -> tuple[float, str, bool]:
+def _score_submission(
+    state: MathEnvState, tool_calls: Sequence[Mapping[str, Any]]
+) -> tuple[float, str, bool]:
     if not tool_calls:
         return REWARD_NEGATIVE_NO_TOOL, "missing_tool_call", False
     call = tool_calls[0]
@@ -374,14 +380,61 @@ def _score_submission(state: MathEnvState, tool_calls: Sequence[Mapping[str, Any
     if not answer:
         return REWARD_NEGATIVE_NO_ANSWER, "blank_answer", False
     is_correct = answer == state.answer
-    return (REWARD_POSITIVE if is_correct else 0.0), ("correct" if is_correct else "incorrect"), is_correct
+    return (
+        (REWARD_POSITIVE if is_correct else 0.0),
+        ("correct" if is_correct else "incorrect"),
+        is_correct,
+    )
 
 
 math_router = APIRouter()
 
 
+def _preview_tool_calls(tool_calls: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
+    """Return a compact, log-friendly preview of tool calls.
+
+    Truncates long fields to avoid noisy logs and leaking excessive content.
+    """
+    preview: list[dict[str, Any]] = []
+    for call in list(tool_calls or [])[:3]:
+        args = dict(call.get("args") or {})
+        answer = str(args.get("answer") or "")
+        # Hard truncate to keep logs compact
+        answer_short = answer[:120] + ("…" if len(answer) > 120 else "")
+        preview.append(
+            {
+                "tool": call.get("tool"),
+                "answer": answer_short,
+            }
+        )
+    return preview
+
+
+def _event_and_outcome_components(
+    tool_calls: Sequence[Mapping[str, Any]], *, correct: bool, reward: float
+) -> dict[str, float]:
+    """Approximate component-wise scores for RL-style logs.
+
+    - env:     task-level scalar reward (our single-step outcome)
+    - rubric_event: 1.0 if a valid tool call with non-empty answer was made else 0.0
+    - rubric_outcome: 1.0 if final answer was correct else 0.0
+    """
+    has_valid_tool = False
+    if tool_calls:
+        first = tool_calls[0] or {}
+        if str(first.get("tool") or "") == TOOL_NAME:
+            args = first.get("args") or {}
+            ans = str(args.get("answer") or "").strip()
+            has_valid_tool = bool(ans)
+    return {
+        "env": float(reward),
+        "rubric_event": 1.0 if has_valid_tool else 0.0,
+        "rubric_outcome": 1.0 if bool(correct) else 0.0,
+    }
+
+
 @math_router.post("/env/math/initialize")
-async def initialize_env(request: Request, payload: InitializePayload) -> Dict[str, Any]:
+async def initialize_env(request: Request, payload: InitializePayload) -> dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
     split = str(payload.config.get("split") or DEFAULT_SPLIT)
     seed = payload.seed
@@ -397,7 +450,7 @@ async def initialize_env(request: Request, payload: InitializePayload) -> Dict[s
 
 
 @math_router.post("/env/math/step")
-async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
+async def step_env(request: Request, payload: dict[str, Any]) -> dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
     env_id = str(payload.get("env_id") or "")
     if not env_id:
@@ -410,6 +463,26 @@ async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
     action = payload.get("action") or {}
     tool_calls = action.get("tool_calls") or payload.get("tool_calls") or []
     reward, status, correct = _score_submission(state, tool_calls)
+    with contextlib.suppress(Exception):
+        print(
+            "[MATH_STEP] env_id=",
+            state.env_id,
+            " split=",
+            state.split,
+            " index=",
+            state.index,
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            " reward=",
+            reward,
+            " status=",
+            status,
+            " correct=",
+            correct,
+            " components=",
+            _event_and_outcome_components(tool_calls, correct=correct, reward=reward),
+            flush=True,
+        )
     state.done = True
 
     observation = _observation_from_state(state)
@@ -427,7 +500,7 @@ async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
 
 
 @math_router.post("/env/math/terminate")
-async def terminate_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
+async def terminate_env(request: Request, payload: dict[str, Any]) -> dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
     env_id = str(payload.get("env_id") or "")
     if env_id:
@@ -448,7 +521,9 @@ def _resolve_inference_url(base_url: str) -> str:
     return f"{normalized}/v1/chat/completions"
 
 
-async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping[str, Any]) -> tuple[list[Dict[str, Any]], Dict[str, Any]]:
+async def _call_inference(
+    policy_config: Mapping[str, Any], observation: Mapping[str, Any]
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
     inference_url = str(policy_config.get("inference_url") or "").rstrip("/")
     if not inference_url:
         raise RuntimeError("policy.config.inference_url required for rollout")
@@ -480,7 +555,7 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
         },
     ]
 
-    payload: Dict[str, Any] = {
+    payload: dict[str, Any] = {
         "model": model,
         "messages": messages,
         "tools": [
@@ -549,7 +624,7 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
             function = call.get("function") or {}
             name = function.get("name")
             arguments = function.get("arguments")
-            parsed_args: Dict[str, Any]
+            parsed_args: dict[str, Any]
             if isinstance(arguments, str):
                 try:
                     import json
@@ -562,6 +637,15 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
             else:
                 parsed_args = {}
             tool_calls.append({"tool": name, "args": parsed_args})
+    # Lightweight provider-side logging
+    with contextlib.suppress(Exception):
+        print(
+            "[MATH_INFER] model=",
+            model,
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            flush=True,
+        )
     return tool_calls, data
 
 
@@ -576,11 +660,13 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         "index": sample["index"],
     }
 
-    tool_calls: list[Dict[str, Any]] = []
-    inference_payload: Dict[str, Any] | None = None
-    error_info: Dict[str, Any] = {}
+    tool_calls: list[dict[str, Any]] = []
+    inference_payload: dict[str, Any] | None = None
+    error_info: dict[str, Any] = {}
     try:
-        tool_calls, inference_payload = await _call_inference(request.policy.config or {}, observation)
+        tool_calls, inference_payload = await _call_inference(
+            request.policy.config or {}, observation
+        )
     except HTTPException as http_err:
         tool_calls = []
         error_info = {"error": http_err.detail, "code": http_err.status_code}
@@ -600,6 +686,28 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         tool_calls,
     )
 
+    # Log a concise summary so we can debug reward=0 issues in production
+    with contextlib.suppress(Exception):
+        print(
+            "[MATH_ROLLOUT] run=",
+            request.run_id,
+            " split=",
+            sample["split"],
+            " index=",
+            sample["index"],
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            " reward=",
+            reward,
+            " status=",
+            status,
+            " correct=",
+            correct,
+            " components=",
+            _event_and_outcome_components(tool_calls, correct=correct, reward=reward),
+            flush=True,
+        )
+
     step = RolloutStep(
         obs=observation,
         tool_calls=tool_calls,
@@ -610,6 +718,7 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
             "status": status,
             "correct": correct,
             "raw_solution": sample["raw_solution"],
+            "tool_call_preview": _preview_tool_calls(tool_calls),
             **error_info,
         },
     )
@@ -634,6 +743,34 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         details={"status": status, "correct": correct},
     )
 
+    # Include a minimal trace when requested or tracing is enabled via env
+    include_trace = bool(
+        (request.record and getattr(request.record, "return_trace", False))
+        or os.getenv("TASKAPP_TRACING_ENABLED")
+    )
+    trace_payload = None
+    if include_trace:
+        try:
+            # Minimal structured trace for assertions
+            trace_payload = {
+                "session_id": str(uuid.uuid4()),
+                "events_count": 1,
+                "decision_rewards": [reward],
+                "lm_calls": (
+                    [{"prompt": str(observation.get("problem", "")), "response": str(tool_calls)}]
+                    if tool_calls
+                    else []
+                ),
+                "metadata": {
+                    "env": "math_single_step",
+                    "split": sample["split"],
+                    "index": sample["index"],
+                    "status": status,
+                },
+            }
+        except Exception:
+            trace_payload = None
+
     return RolloutResponse(
         run_id=request.run_id,
         trajectories=[trajectory],
@@ -641,7 +778,7 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         metrics=metrics,
         aborted=False,
         ops_executed=2,
-        trace=None,
+        trace=trace_payload,
     )
 
 
@@ -663,7 +800,7 @@ def build_dataset() -> tuple[TaskDatasetRegistry, MathDataset]:
 def _base_task_info() -> TaskInfo:
     return TaskInfo(
         task={"id": "math_single_step", "name": "Math Single Step", "version": "1.0.0"},
-        environments=["math"],
+        environment="math",
         action_space={
             "type": "tool_call",
             "tools": [
@@ -739,7 +876,7 @@ EVENTS_RUBRIC: Rubric = cast(
 )
 
 
-def describe_taskset(dataset: MathDataset) -> Dict[str, Any]:
+def describe_taskset(dataset: MathDataset) -> dict[str, Any]:
     return {
         **MATH_DATASET_SPEC.model_dump(),
         "hf_dataset": DATASET_NAME,
@@ -754,7 +891,7 @@ def provide_task_instances(dataset: MathDataset, seeds: Sequence[int]) -> Iterab
         sample = dataset.sample(split=DEFAULT_SPLIT, index=seed)
         yield TaskInfo(
             task=info.task,
-            environments=info.environments,
+            environment=info.environment,
             action_space=info.action_space,
             observation={**info.observation, "sample_index": sample["index"]},
             dataset={
@@ -775,10 +912,12 @@ def build_config() -> TaskAppConfig:
 
     tracing_enabled = tracing_env_enabled()
     tracing_db_url = resolve_tracing_db_url()
-    tracer_factory = build_tracer_factory(SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url)
+    tracer_factory = build_tracer_factory(
+        SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url
+    )
     sft_output_dir = resolve_sft_output_dir()
 
-    app_state: Dict[str, Any] = {
+    app_state: dict[str, Any] = {
         "math_dataset": dataset,
         "math_env_manager": MathEnvironmentManager(dataset),
         "tracing_enabled": tracing_enabled,
@@ -823,7 +962,6 @@ register_task_app(
         description="Single-step math reasoning task app using EleutherAI/math dataset.",
         config_factory=build_config,
         aliases=("math-rl",),
-        env_files=("examples/rl/.env",),
         modal=ModalDeploymentConfig(
             app_name="synth-math-single-step",
             pip_packages=(

examples/rl/task_app/math_task_app.py

@@ -0,0 +1,111 @@
+"""Legacy entrypoint for the math single-step task app."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
+from synth_ai.task.server import create_task_app, run_task_app
+
+from .math_single_step import build_config
+
+
+def fastapi_app():
+    """Return a FastAPI application for hosting the math task app."""
+
+    app = create_task_app(build_config())
+
+    # Replace default health endpoints with auth-tolerant handlers.
+    filtered_routes = []
+    for route in app.router.routes:
+        path = getattr(route, "path", None)
+        methods = getattr(route, "methods", set()) or set()
+        if path in {"/health", "/health/rollout"} and "GET" in methods:
+            continue
+        filtered_routes.append(route)
+    app.router.routes = filtered_routes
+
+    def _log_env_key_prefix(source: str, env_key: str | None) -> str | None:
+        if not env_key:
+            return None
+        prefix = env_key[: max(1, len(env_key) // 2)]
+        print(f"[{source}] expected ENVIRONMENT_API_KEY prefix: {prefix}")
+        return prefix
+
+    @app.get("/health")
+    async def health(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"status": "healthy", "authorized": True}
+
+    @app.get("/health/rollout")
+    async def health_rollout(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health/rollout", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"ok": True, "authorized": True}
+
+    @app.exception_handler(RequestValidationError)
+    async def _on_validation_error(request: Request, exc: RequestValidationError):
+        try:
+            hdr = request.headers
+            snapshot = {
+                "path": str(request.url.path),
+                "have_x_api_key": bool(hdr.get("x-api-key")),
+                "have_x_api_keys": bool(hdr.get("x-api-keys")),
+                "have_authorization": bool(hdr.get("authorization")),
+                "errors": exc.errors()[:5],
+            }
+            print("[422] validation", snapshot, flush=True)
+        except Exception:
+            pass
+        return JSONResponse(
+            status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]}
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the math single-step task app locally")
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=8101)
+    parser.add_argument("--reload", action="store_true", help="Enable uvicorn autoreload")
+    parser.add_argument(
+        "--env-file",
+        action="append",
+        default=[],
+        help="Path to .env file to load (can be specified multiple times)",
+    )
+    args = parser.parse_args()
+
+    run_task_app(
+        build_config,
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+        env_files=args.env_file or [],
+    )

examples/run_crafter_demo.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Run a Crafter agent demo with Gemini
+# This script demonstrates a reactive agent in the Crafter environment
+
+echo "🚀 Starting Crafter agent demo with Gemini 1.5 Flash..."
+echo "Make sure the synth-ai service is running: uvx synth-ai deploy --runtime uvicorn"
+echo ""
+
+uv run python -m synth_ai.environments.examples.crafter_classic.agent_demos.test_crafter_react_agent --model gemini-1.5-flash

examples/sdk_prompt_learning_example.py

@@ -0,0 +1,55 @@
+"""Example: Running prompt learning jobs programmatically.
+
+This example shows how to use the SDK to run MIPRO or GEPA optimization
+in a Python script instead of via CLI.
+"""
+
+import os
+from synth_ai.api.train.prompt_learning import PromptLearningJob
+
+
+def main() -> None:
+    """Run a prompt learning job programmatically."""
+    # Initialize job from config
+    job = PromptLearningJob.from_config(
+        config_path="examples/blog_posts/gepa/configs/banking77_gepa_local.toml",
+        backend_url=os.environ.get("BACKEND_BASE_URL", "https://api.usesynth.ai"),
+        api_key=os.environ["SYNTH_API_KEY"],
+    )
+    
+    # Submit job
+    print("Submitting job...")
+    job_id = job.submit()
+    print(f"Job submitted: {job_id}")
+    
+    # Poll until complete with status callback
+    def on_status(status: dict) -> None:
+        print(f"Status: {status.get('status', 'unknown')}")
+        if 'best_score' in status:
+            print(f"Best score so far: {status['best_score']}")
+    
+    print("Polling until complete...")
+    final_status = job.poll_until_complete(
+        timeout=3600.0,
+        interval=5.0,
+        on_status=on_status,
+    )
+    
+    print(f"\nJob completed with status: {final_status.get('status')}")
+    
+    # Get results
+    print("\nFetching results...")
+    results = job.get_results()
+    
+    print(f"Best score: {results.get('best_score')}")
+    print(f"Top prompts: {len(results.get('top_prompts', []))}")
+    
+    # Get best prompt text
+    best_prompt = job.get_best_prompt_text(rank=1)
+    if best_prompt:
+        print(f"\nBest prompt:\n{best_prompt}")
+
+
+if __name__ == "__main__":
+    main()
+