synth-ai 0.2.9.dev4__py3-none-any.whl → 0.2.9.dev7__py3-none-any.whl

examples/rl/run_rl_and_save.py

@@ -24,10 +24,18 @@ def _load_toml(path: Path) -> Dict[str, Any]:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Create math RL job via backend RL endpoint")
-    parser.add_argument("--backend", default=os.getenv("BACKEND_BASE_URL", "http://localhost:8000/api"))
+    parser.add_argument(
+        "--backend", default=os.getenv("BACKEND_BASE_URL", "http://localhost:8000/api")
+    )
     parser.add_argument("--config", required=True, help="Path to RL TOML config")
-    parser.add_argument("--task-url", default=os.getenv("TASK_APP_URL", ""), help="Override task service URL")
-    parser.add_argument("--idempotency", default=os.getenv("RL_IDEMPOTENCY_KEY", ""), help="Optional Idempotency-Key header")
+    parser.add_argument(
+        "--task-url", default=os.getenv("TASK_APP_URL", ""), help="Override task service URL"
+    )
+    parser.add_argument(
+        "--idempotency",
+        default=os.getenv("RL_IDEMPOTENCY_KEY", ""),
+        help="Optional Idempotency-Key header",
+    )
     args = parser.parse_args()
 
     cfg_path = Path(args.config).expanduser()
@@ -35,16 +43,26 @@ def main() -> None:
 
     services = cfg.get("services") if isinstance(cfg.get("services"), dict) else {}
 
-    task_url = (args.task_url or "").strip() or (os.getenv("TASK_APP_URL") or "").strip() or (services.get("task_url") or "").strip()
+    task_url = (
+        (args.task_url or "").strip()
+        or (os.getenv("TASK_APP_URL") or "").strip()
+        or (services.get("task_url") or "").strip()
+    )
     if not task_url:
-        print("Missing task service URL. Provide --task-url or set TASK_APP_URL or services.task_url in TOML", file=sys.stderr)
+        print(
+            "Missing task service URL. Provide --task-url or set TASK_APP_URL or services.task_url in TOML",
+            file=sys.stderr,
+        )
         sys.exit(2)
 
     model_cfg = cfg.get("model") if isinstance(cfg.get("model"), dict) else {}
     has_source = bool((model_cfg.get("source") or "").strip())
     has_base = bool((model_cfg.get("base") or "").strip())
     if has_source == has_base:
-        print("Model section must specify exactly one of [model].source or [model].base", file=sys.stderr)
+        print(
+            "Model section must specify exactly one of [model].source or [model].base",
+            file=sys.stderr,
+        )
         sys.exit(2)
 
     payload: Dict[str, Any] = {
@@ -91,4 +109,3 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
-

examples/rl/task_app/math_single_step.py

@@ -40,7 +40,9 @@ from synth_ai.tracing_v3.session_tracer import SessionTracer
 
 REPO_ROOT = Path(__file__).resolve().parents[3]
 
-_modal_volume_candidate = Path(os.getenv("MATH_MODAL_DATASET_DIR", "/modal_volumes/math_dataset")).expanduser()
+_modal_volume_candidate = Path(
+    os.getenv("MATH_MODAL_DATASET_DIR", "/modal_volumes/math_dataset")
+).expanduser()
 _modal_volume_root: Optional[Path] = None
 try:
     _modal_volume_candidate.mkdir(parents=True, exist_ok=True)
@@ -55,7 +57,9 @@ if _modal_volume_root is not None:
     local_dataset_dir.mkdir(parents=True, exist_ok=True)
     os.environ.setdefault("MATH_DATASET_LOCAL_DIR", str(local_dataset_dir))
 else:
-    hf_cache_path = Path(os.getenv("MATH_DATASET_CACHE_DIR", str(REPO_ROOT / ".cache" / "hf-datasets")) ).expanduser()
+    hf_cache_path = Path(
+        os.getenv("MATH_DATASET_CACHE_DIR", str(REPO_ROOT / ".cache" / "hf-datasets"))
+    ).expanduser()
 
 hf_cache_path.mkdir(parents=True, exist_ok=True)
 os.environ.setdefault("MATH_DATASET_CACHE_DIR", str(hf_cache_path))
@@ -203,7 +207,9 @@ class MathDataset:
         if split not in self._cache:
             local_file = self._local_file_for_split(split)
             if local_file is not None:
-                dataset = load_dataset("json", data_files=str(local_file), cache_dir=str(HF_DATASETS_CACHE))
+                dataset = load_dataset(
+                    "json", data_files=str(local_file), cache_dir=str(HF_DATASETS_CACHE)
+                )
                 self._cache[split] = dataset["train"]
             else:
                 try:
@@ -301,9 +307,7 @@ class MathDataset:
             except Exception as exc:
                 errors.append(f"{split}: {exc}")
         if errors:
-            raise RuntimeError(
-                "Dataset preparation failed:\n" + "\n".join(errors)
-            )
+            raise RuntimeError("Dataset preparation failed:\n" + "\n".join(errors))
 
 
 @dataclass
@@ -362,7 +366,9 @@ def _observation_from_state(state: MathEnvState) -> Dict[str, Any]:
     }
 
 
-def _score_submission(state: MathEnvState, tool_calls: Sequence[Mapping[str, Any]]) -> tuple[float, str, bool]:
+def _score_submission(
+    state: MathEnvState, tool_calls: Sequence[Mapping[str, Any]]
+) -> tuple[float, str, bool]:
     if not tool_calls:
         return REWARD_NEGATIVE_NO_TOOL, "missing_tool_call", False
     call = tool_calls[0]
@@ -374,12 +380,59 @@ def _score_submission(state: MathEnvState, tool_calls: Sequence[Mapping[str, Any
     if not answer:
         return REWARD_NEGATIVE_NO_ANSWER, "blank_answer", False
     is_correct = answer == state.answer
-    return (REWARD_POSITIVE if is_correct else 0.0), ("correct" if is_correct else "incorrect"), is_correct
+    return (
+        (REWARD_POSITIVE if is_correct else 0.0),
+        ("correct" if is_correct else "incorrect"),
+        is_correct,
+    )
 
 
 math_router = APIRouter()
 
 
+def _preview_tool_calls(tool_calls: Sequence[Mapping[str, Any]]) -> list[Dict[str, Any]]:
+    """Return a compact, log-friendly preview of tool calls.
+
+    Truncates long fields to avoid noisy logs and leaking excessive content.
+    """
+    preview: list[Dict[str, Any]] = []
+    for call in list(tool_calls or [])[:3]:
+        args = dict(call.get("args") or {})
+        answer = str(args.get("answer") or "")
+        # Hard truncate to keep logs compact
+        answer_short = answer[:120] + ("…" if len(answer) > 120 else "")
+        preview.append(
+            {
+                "tool": call.get("tool"),
+                "answer": answer_short,
+            }
+        )
+    return preview
+
+
+def _event_and_outcome_components(
+    tool_calls: Sequence[Mapping[str, Any]], *, correct: bool, reward: float
+) -> Dict[str, float]:
+    """Approximate component-wise scores for RL-style logs.
+
+    - env:     task-level scalar reward (our single-step outcome)
+    - rubric_event: 1.0 if a valid tool call with non-empty answer was made else 0.0
+    - rubric_outcome: 1.0 if final answer was correct else 0.0
+    """
+    has_valid_tool = False
+    if tool_calls:
+        first = tool_calls[0] or {}
+        if str(first.get("tool") or "") == TOOL_NAME:
+            args = first.get("args") or {}
+            ans = str(args.get("answer") or "").strip()
+            has_valid_tool = bool(ans)
+    return {
+        "env": float(reward),
+        "rubric_event": 1.0 if has_valid_tool else 0.0,
+        "rubric_outcome": 1.0 if bool(correct) else 0.0,
+    }
+
+
 @math_router.post("/env/math/initialize")
 async def initialize_env(request: Request, payload: InitializePayload) -> Dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
@@ -410,6 +463,28 @@ async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
     action = payload.get("action") or {}
     tool_calls = action.get("tool_calls") or payload.get("tool_calls") or []
     reward, status, correct = _score_submission(state, tool_calls)
+    try:
+        print(
+            "[MATH_STEP] env_id=",
+            state.env_id,
+            " split=",
+            state.split,
+            " index=",
+            state.index,
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            " reward=",
+            reward,
+            " status=",
+            status,
+            " correct=",
+            correct,
+            " components=",
+            _event_and_outcome_components(tool_calls, correct=correct, reward=reward),
+            flush=True,
+        )
+    except Exception:
+        pass
     state.done = True
 
     observation = _observation_from_state(state)
@@ -448,7 +523,9 @@ def _resolve_inference_url(base_url: str) -> str:
     return f"{normalized}/v1/chat/completions"
 
 
-async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping[str, Any]) -> tuple[list[Dict[str, Any]], Dict[str, Any]]:
+async def _call_inference(
+    policy_config: Mapping[str, Any], observation: Mapping[str, Any]
+) -> tuple[list[Dict[str, Any]], Dict[str, Any]]:
     inference_url = str(policy_config.get("inference_url") or "").rstrip("/")
     if not inference_url:
         raise RuntimeError("policy.config.inference_url required for rollout")
@@ -562,6 +639,17 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
             else:
                 parsed_args = {}
             tool_calls.append({"tool": name, "args": parsed_args})
+    # Lightweight provider-side logging
+    try:
+        print(
+            "[MATH_INFER] model=",
+            model,
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            flush=True,
+        )
+    except Exception:
+        pass
     return tool_calls, data
 
 
@@ -580,7 +668,9 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
     inference_payload: Dict[str, Any] | None = None
     error_info: Dict[str, Any] = {}
     try:
-        tool_calls, inference_payload = await _call_inference(request.policy.config or {}, observation)
+        tool_calls, inference_payload = await _call_inference(
+            request.policy.config or {}, observation
+        )
     except HTTPException as http_err:
         tool_calls = []
         error_info = {"error": http_err.detail, "code": http_err.status_code}
@@ -600,6 +690,30 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         tool_calls,
     )
 
+    # Log a concise summary so we can debug reward=0 issues in production
+    try:
+        print(
+            "[MATH_ROLLOUT] run=",
+            request.run_id,
+            " split=",
+            sample["split"],
+            " index=",
+            sample["index"],
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            " reward=",
+            reward,
+            " status=",
+            status,
+            " correct=",
+            correct,
+            " components=",
+            _event_and_outcome_components(tool_calls, correct=correct, reward=reward),
+            flush=True,
+        )
+    except Exception:
+        pass
+
     step = RolloutStep(
         obs=observation,
         tool_calls=tool_calls,
@@ -610,6 +724,7 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
             "status": status,
             "correct": correct,
             "raw_solution": sample["raw_solution"],
+            "tool_call_preview": _preview_tool_calls(tool_calls),
             **error_info,
         },
     )
@@ -775,7 +890,9 @@ def build_config() -> TaskAppConfig:
 
     tracing_enabled = tracing_env_enabled()
     tracing_db_url = resolve_tracing_db_url()
-    tracer_factory = build_tracer_factory(SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url)
+    tracer_factory = build_tracer_factory(
+        SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url
+    )
     sft_output_dir = resolve_sft_output_dir()
 
     app_state: Dict[str, Any] = {

examples/rl/task_app/math_task_app.py

@@ -40,7 +40,10 @@ def fastapi_app():
     async def health(request: Request):
         env_key = normalize_environment_api_key()
         if not env_key:
-            return JSONResponse(status_code=503, content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"})
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
         if not is_api_key_header_authorized(request):
             prefix = _log_env_key_prefix("health", env_key)
             content = {"status": "healthy", "authorized": False}
@@ -53,7 +56,10 @@ def fastapi_app():
     async def health_rollout(request: Request):
         env_key = normalize_environment_api_key()
         if not env_key:
-            return JSONResponse(status_code=503, content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"})
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
         if not is_api_key_header_authorized(request):
             prefix = _log_env_key_prefix("health/rollout", env_key)
             content = {"status": "healthy", "authorized": False}
@@ -76,7 +82,9 @@ def fastapi_app():
             print("[422] validation", snapshot, flush=True)
         except Exception:
             pass
-        return JSONResponse(status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]})
+        return JSONResponse(
+            status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]}
+        )
 
     return app