synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py

@@ -466,11 +466,20 @@ async def step_policy(
 
             if tracing_context is not None:
                 try:
+                    print(
+                        f"[TRACE_DEBUG] record_policy_prompts sys={len(system_prompt_records)} user={len(user_prompt_records)}",
+                        flush=True,
+                    )
                     await tracing_context.record_policy_prompts(
                         system_prompt_records, user_prompt_records
                     )
                 except Exception as exc:
                     logger.debug(f"TRACING_PROMPTS_FAIL: {exc}")
+            else:
+                print(
+                    f"[TRACE_DEBUG] Missing tracing context on policy step; policy_id={request.policy_id}",
+                    flush=True,
+                )
 
             # Create inference client (choose API key by target provider)
             # Require inference_url to be set explicitly by the rollout policy config.
@@ -492,7 +501,11 @@ async def step_policy(
                 if isinstance(target_url, str):
                     low_url = target_url.lower()
                     # Proxy endpoints should not receive a bearer; the server-side proxy holds the vendor key
-                    if "/proxy/groq" in low_url or "/proxy/openai" in low_url:
+                    if (
+                        "/proxy/groq" in low_url
+                        or "/proxy/openai" in low_url
+                        or "/proxy/v1" in low_url
+                    ):
                         api_key_override = None
                     elif "openai.com" in low_url:
                         api_key_override = _os.getenv("OPENAI_API_KEY") or getattr(
@@ -692,9 +705,10 @@ async def step_policy(
                 "sokoban-react",
                 "crafter-react",
             ) and getattr(policy, "use_tools", True):
-                req_tools = meta["inference_request"]["tools"]
-                req_tool_choice = meta["inference_request"]["tool_choice"]
-                req_stop_after = meta["inference_request"]["stop_after_tool_calls"]
+                inf_req = meta.get("inference_request", {})
+                req_tools = inf_req.get("tools")
+                req_tool_choice = inf_req.get("tool_choice")
+                req_stop_after = inf_req.get("stop_after_tool_calls")
                 logger.info(
                     f"TOOLCALL_CONFIG: policy={policy_name} tools_present={bool(req_tools)} tool_choice={req_tool_choice} stop_after={req_stop_after}"
                 )
@@ -703,6 +717,8 @@ async def step_policy(
                         status_code=500,
                         detail=f"TOOLCALL_ASSERTION_FAIL: Missing tools or tool_choice!=required for policy {policy_name}",
                     )
+                if req_stop_after is None:
+                    inf_req["stop_after_tool_calls"] = 1
 
             # Call inference service with retries for Flash cold-start (503)
             import time as _t
@@ -951,6 +967,23 @@ async def step_policy(
                 except Exception as exc:
                     logger.debug(f"TRACING_LLM_FAIL: {exc}")
 
+        if not tool_calls:
+            preview = ""
+            try:
+                preview = str(meta.get("raw_response") or "")[:400]
+            except Exception:
+                preview = "<unavailable>"
+            logger.error(
+                {
+                    "rollout.policy_step": True,
+                    "policy_id": request.policy_id,
+                    "error": "no_tool_calls",
+                    "inference_url": meta.get("inference_url"),
+                    "raw_preview": preview,
+                }
+            )
+            raise RuntimeError("Policy step produced no tool calls; inference response unusable.")
+
         return PolicyStepResponse(
             tool_calls=tool_calls,
             meta=meta,

examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py

@@ -223,6 +223,7 @@ class RolloutTracingContext:
         ).lower()
         self.return_trace = bool(getattr(request.record, "return_trace", False))
         self.sft_output_dir = getattr(fastapi_request.app.state, "sft_output_dir", None)
+        print(f"[TRACE_DEBUG] RolloutTracingContext init: trace_format={self.trace_format} return_trace={self.return_trace}", flush=True)
         self.session_trace = None
         self.metadata_updates: dict[str, Any] = {}
         self.policy_name = request.policy.policy_name or ""
@@ -244,19 +245,24 @@ class RolloutTracingContext:
 
     async def start_session(self) -> None:
         if not self.enabled or self.tracer is None:
+            print("[TRACE_DEBUG] start_session skipped: tracer disabled", flush=True)
             return
         try:
             await self.tracer.initialize()
+            print("[TRACE_DEBUG] tracer initialized", flush=True)
         except Exception as exc:
             logger.debug("TRACING_INIT_FAIL: %s", exc)
+            # Hard fail: tracing requested but cannot initialize
+            raise
         try:
             await self.tracer.start_session(
                 session_id=self.run_id, metadata=dict(self.metadata_base)
             )
+            print(f"[TRACE_DEBUG] start_session succeeded for run_id={self.run_id}", flush=True)
         except Exception as exc:
             logger.warning("TRACING_START_FAIL: %s", exc)
-            self.enabled = False
-            self.tracer = None
+            # Hard fail: tracing requested but cannot start session
+            raise
 
     async def start_decision(self, turn_number: int) -> None:
         self.current_turn = turn_number
@@ -317,6 +323,9 @@ class RolloutTracingContext:
                 )
             except Exception as exc:
                 logger.debug("TRACING_USER_MSG_FAIL: %s", exc)
+        if self.tracer and self.tracer._current_trace:
+            msg_count = len(self.tracer._current_trace.markov_blanket_message_history)
+            print(f"[TRACE_DEBUG] After record_policy_prompts: {msg_count} messages", flush=True)
 
     def _content_to_text(self, content: Any) -> str:
         if isinstance(content, str):
@@ -395,6 +404,11 @@ class RolloutTracingContext:
                     message_type="policy_tool_call",
                     metadata=self._message_metadata(),
                 )
+                if self.tracer._current_trace:
+                    print(
+                        f"[TRACE_DEBUG] After tool invocation: messages={len(self.tracer._current_trace.markov_blanket_message_history)}",
+                        flush=True,
+                    )
             except Exception as exc:
                 logger.debug("TRACING_TOOL_MSG_FAIL: %s", exc)
 
@@ -664,12 +678,24 @@ class RolloutTracingContext:
             except Exception as exc:
                 logger.debug("TRACING_OUTCOME_FAIL: %s", exc)
             try:
+                if self.tracer._current_trace:
+                    msg_count = len(self.tracer._current_trace.markov_blanket_message_history)
+                    print(f"[TRACE_DEBUG] Before end_session: {msg_count} messages in trace", flush=True)
                 self.session_trace = await self.tracer.end_session()
                 if self.session_trace is not None:
                     self.session_trace.metadata.update(self.metadata_updates)
+                    print(
+                        f"[TRACE_DEBUG] Session ended successfully, session_id={self.session_trace.session_id}",
+                        flush=True,
+                    )
+                    print(
+                        f"[TRACE_DEBUG] session_trace.metadata keys: {list(self.session_trace.metadata.keys())}",
+                        flush=True,
+                    )
             except Exception as exc:
                 logger.debug("TRACING_END_SESSION_FAIL: %s", exc)
                 self.session_trace = None
+                print(f"[TRACE_DEBUG] end_session failed for run_id={self.run_id}: {exc}", flush=True)
             with contextlib.suppress(Exception):
                 await self.tracer.close()
 
@@ -700,9 +726,13 @@ class RolloutTracingContext:
     def build_trace_payload(self, session_trace: Any) -> dict[str, Any] | None:
         if not self.return_trace or session_trace is None:
             return None
-        if self.trace_format == "full":
+        if self.trace_format in ("full", "structured"):
             payload = session_trace.to_dict()
             payload.setdefault("metadata", {}).update(self.metadata_updates)
+            print(
+                f"[TRACE_DEBUG] build_trace_payload returning structured trace with messages={len(payload.get('markov_blanket_message_history') or [])}",
+                flush=True,
+            )
             return payload
         metadata = dict(session_trace.metadata)
         metadata.update(self.metadata_updates)

examples/warming_up_to_rl/task_app/synth_envs_hosted/utils.py

@@ -1,6 +1,7 @@
 """Utility functions for the task service."""
 
 from typing import Any
+from urllib.parse import urlparse, urlunparse
 
 import numpy as np
 
@@ -60,3 +61,69 @@ def sanitize_observation(observation: dict[str, Any]) -> dict[str, Any]:
             sanitized[key] = convert_numpy_to_python(value)
 
     return sanitized
+
+
+_CHAT_COMPLETIONS_SUFFIX = "/v1/chat/completions"
+
+
+def force_normalize_chat_completions_url(raw_url: Any) -> Any:
+    """
+    Convert ANY malformed inference URL into the correct chat-completions form.
+    Ensures path ends with /v1/chat/completions and that query has no '/' segments.
+    """
+    if not isinstance(raw_url, str):
+        return raw_url
+    url = raw_url.strip()
+    if not url:
+        return raw_url
+
+    parsed = urlparse(url)
+    path = (parsed.path or "").rstrip("/")
+    query = parsed.query or ""
+
+    # If query contains a path, extract and repair
+    if query and "/" in query:
+        before_slash, after_slash = query.split("/", 1)
+        cut_positions = [i for i in [after_slash.find("&"), after_slash.find("?")] if i >= 0]
+        cut = min(cut_positions) if cut_positions else len(after_slash)
+        path_from_query = "/" + after_slash[:cut]
+        extra_query = after_slash[cut + 1 :] if cut < len(after_slash) else ""
+        merged_query = before_slash if before_slash else ""
+        if extra_query:
+            merged_query = f"{merged_query}&{extra_query}" if merged_query else extra_query
+        final_path = (
+            path_from_query
+            if path_from_query.startswith(_CHAT_COMPLETIONS_SUFFIX)
+            else f"{path_from_query.rstrip('/')}{_CHAT_COMPLETIONS_SUFFIX}"
+        )
+        parsed = parsed._replace(path=final_path, query=merged_query)
+        url = urlunparse(parsed)
+        parsed = urlparse(url)
+        path = parsed.path or ""
+        query = parsed.query or ""
+
+    # Ensure path suffix
+    if not path.endswith(_CHAT_COMPLETIONS_SUFFIX):
+        new_path = f"{path}{_CHAT_COMPLETIONS_SUFFIX}" if path else _CHAT_COMPLETIONS_SUFFIX
+        parsed = parsed._replace(path=new_path)
+        url = urlunparse(parsed)
+        parsed = urlparse(url)
+        path = parsed.path or ""
+        query = parsed.query or ""
+
+    # Last-resort: strip any '/' from query
+    if query and "/" in query:
+        safe_query = query.split("/")[0]
+        parsed = parsed._replace(query=safe_query)
+        url = urlunparse(parsed)
+
+    return url
+
+
+def ensure_chat_completions_url(raw_url: Any, mode: Any = None) -> Any:
+    """
+    Mode-aware normalizer (RL/EVAL) that returns a valid chat completions URL and
+    preserves existing query parameters.
+    """
+    # For now reuse force normalizer in both modes to guarantee correctness
+    return force_normalize_chat_completions_url(raw_url)

examples/workflows/math_rl/configs/rl_from_base_qwen.toml

@@ -1,8 +1,15 @@
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
 [services]
 task_url = "https://your-math-task.modal.run"
 
 [model]
 base = "Qwen/Qwen3-4B"
+trainer_mode = "full"
+label = "math-single-step-qwen3-4b"
 
 [policy]
 model = "Qwen/Qwen3-4B"
@@ -18,6 +25,8 @@ evaluation_split = "validation"
 evaluation_episodes = 256
 
 [training]
+num_epochs = 1
+iterations_per_epoch = 20
 max_turns = 1
 ops = ["agent", "env"]
 batch_size = 128
@@ -31,5 +40,23 @@ learning_rate = 5e-6
 gpu_type = "A10G"
 gpu_count = 4
 
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 2
+gpus_for_training = 2
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[rollout]
+env_name = "math"
+policy_name = "math-single-step"
+max_turns = 1
+episodes_per_batch = 256
+
+[evaluation]
+instances = 256
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4]
+
 [tags]
 experiment = "math_single_step"

examples/workflows/math_rl/configs/rl_from_base_qwen17.toml

@@ -8,6 +8,8 @@ task_url = "http://localhost:8101"
 
 [model]
 base = "Qwen/Qwen3-1.7B"
+trainer_mode = "full"
+label = "math-single-step-qwen3-1.7b"
 
 [policy]
 model = "Qwen/Qwen3-1.7B"
@@ -23,6 +25,8 @@ evaluation_split = "validation"
 evaluation_episodes = 50
 
 [training]
+num_epochs = 1
+iterations_per_epoch = 20
 max_turns = 1
 ops = ["agent", "env"]
 batch_size = 2
@@ -61,9 +65,11 @@ health_max_wait_s = 180
 health_interval_ms = 300
 
 [rollout]
+env_name = "math"
 policy_name = "math-single-step"
 max_turns = 1
 episodes_per_batch = 32  # group_size * batch_size
+task_app_origin_rewards_only = true
 
 [evaluation]
 instances = 32

synth_ai/api/train/builders.py

@@ -33,7 +33,7 @@ try:
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load SFT payload helpers") from exc
 
-from .configs import RLConfig, SFTConfig
+from .configs import PromptLearningConfig, RLConfig, SFTConfig
 from .supported_algos import (
     AlgorithmValidationError,
     ensure_model_supported_for_algorithm,
@@ -56,6 +56,12 @@ class SFTBuildResult:
     validation_file: Path | None
 
 
+@dataclass(slots=True)
+class PromptLearningBuildResult:
+    payload: dict[str, Any]
+    task_url: str
+
+
 def _format_validation_error(path: Path, exc: ValidationError) -> str:
     lines: list[str] = []
     for error in exc.errors():
@@ -74,12 +80,23 @@ def build_rl_payload(
     idempotency: str | None,
     allow_experimental: bool | None = None,
 ) -> RLBuildResult:
+    # Load and validate config with SDK-level checks
+    from synth_ai.api.train.utils import load_toml
+    from synth_ai.cli.commands.train.validation import validate_rl_config
+    
     try:
-        rl_cfg = RLConfig.from_path(config_path)
+        raw_config = load_toml(config_path)
+        validated_config = validate_rl_config(raw_config)  # Adds defaults & validates
+        rl_cfg = RLConfig.from_mapping(validated_config)
     except ValidationError as exc:
         raise click.ClickException(_format_validation_error(config_path, exc)) from exc
 
     data = rl_cfg.to_dict()
+    
+    # Remove smoke section - it's CLI-only and should not be sent to the trainer
+    if "smoke" in data:
+        del data["smoke"]
+    
     # Ensure required [reference] section for backend validators
     try:
         ref_cfg = data.get("reference") if isinstance(data, dict) else None
@@ -110,8 +127,8 @@ def build_rl_payload(
             "Task app URL required (provide --task-url or set services.task_url in TOML)"
         )
 
-    model_source = (model_cfg.source or "").strip()
-    model_base = (model_cfg.base or "").strip()
+    model_source = (model_cfg.source or "").strip() if model_cfg else ""
+    model_base = (model_cfg.base or "").strip() if model_cfg else ""
     override_model = (overrides.get("model") or "").strip()
     if override_model:
         model_source = override_model
@@ -343,9 +360,87 @@ def build_sft_payload(
     return SFTBuildResult(payload=payload, train_file=dataset_path, validation_file=validation_file)
 
 
+def build_prompt_learning_payload(
+    *,
+    config_path: Path,
+    task_url: str | None,
+    overrides: dict[str, Any],
+    allow_experimental: bool | None = None,
+) -> PromptLearningBuildResult:
+    """Build payload for prompt learning job (MIPRO or GEPA)."""
+    import os
+
+    from pydantic import ValidationError
+
+    from .configs.prompt_learning import load_toml
+
+    # SDK-SIDE VALIDATION: Catch errors BEFORE sending to backend
+    from .validators import validate_prompt_learning_config
+    
+    raw_config = load_toml(config_path)
+    validate_prompt_learning_config(raw_config, config_path)
+    
+    try:
+        pl_cfg = PromptLearningConfig.from_path(config_path)
+    except ValidationError as exc:
+        raise click.ClickException(_format_validation_error(config_path, exc)) from exc
+    
+    # Source of truth: TOML only (ignore shell/env and CLI overrides)
+    final_task_url = (pl_cfg.task_app_url or "").strip()
+    
+    if not final_task_url:
+        raise click.ClickException(
+            "Task app URL required (provide --task-url or set prompt_learning.task_app_url in TOML)"
+        )
+    
+    # Get task_app_api_key from config or environment
+    task_app_api_key = (
+        pl_cfg.task_app_api_key
+        or os.environ.get("ENVIRONMENT_API_KEY", "")
+    ).strip()
+    
+    if not task_app_api_key:
+        raise click.ClickException(
+            "Task app API key required (set prompt_learning.task_app_api_key in TOML or ENVIRONMENT_API_KEY env var)"
+        )
+    
+    # Build config dict for backend
+    config_dict = pl_cfg.to_dict()
+    
+    # Ensure task_app_url and task_app_api_key are set
+    pl_section = config_dict.get("prompt_learning", {})
+    if isinstance(pl_section, dict):
+        pl_section["task_app_url"] = final_task_url
+        pl_section["task_app_api_key"] = task_app_api_key
+    else:
+        config_dict["prompt_learning"] = {
+            "task_app_url": final_task_url,
+            "task_app_api_key": task_app_api_key,
+        }
+    
+    # Build payload matching backend API format
+    payload: dict[str, Any] = {
+        "algorithm": pl_cfg.algorithm,
+        "config_body": config_dict,
+        "overrides": overrides.get("overrides", {}),
+        "metadata": overrides.get("metadata", {}),
+        "auto_start": overrides.get("auto_start", True),
+    }
+    
+    backend = overrides.get("backend")
+    if backend:
+        metadata_default: dict[str, Any] = {}
+        metadata = cast(dict[str, Any], payload.setdefault("metadata", metadata_default))
+        metadata["backend_base_url"] = ensure_api_base(str(backend))
+    
+    return PromptLearningBuildResult(payload=payload, task_url=final_task_url)
+
+
 __all__ = [
+    "PromptLearningBuildResult",
     "RLBuildResult",
     "SFTBuildResult",
+    "build_prompt_learning_payload",
     "build_rl_payload",
     "build_sft_payload",
 ]