synth-ai 0.2.8.dev9__tar.gz → 0.2.8.dev11__tar.gz

{synth_ai-0.2.8.dev9/synth_ai.egg-info → synth_ai-0.2.8.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.8.dev9
+Version: 0.2.8.dev11
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT

{synth_ai-0.2.8.dev9 → synth_ai-0.2.8.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synth-ai"
-version = "0.2.8.dev9"
+version = "0.2.8.dev11"
 description = "RL as a service SDK - Core AI functionality and tracing"
 authors = [{name = "Synth AI", email = "josh@usesynth.ai"}]
 license = "MIT"

{synth_ai-0.2.8.dev9 → synth_ai-0.2.8.dev11}/synth_ai/demos/core/cli.py

@@ -1216,6 +1216,59 @@ def cmd_run(args: argparse.Namespace) -> int:
         except Exception:
             print(str(js))
         print("Request body was:\n" + json.dumps(body, indent=2))
+        try:
+            auth_preview = _key_preview(env.synth_api_key or "", "SYNTH_API_KEY (auth)")
+            print(f"[run] {auth_preview}")
+        except Exception:
+            pass
+        try:
+            data_block = body.get("data") if isinstance(body, dict) else None
+            env_key_body = ""
+            if isinstance(data_block, dict):
+                env_key_body = str(data_block.get("environment_api_key") or "")
+            if env_key_body:
+                print(f"[run] {_key_preview(env_key_body, 'environment_api_key (body)')}")
+        except Exception:
+            pass
+        try:
+            current_env_key = env.env_api_key or ""
+            if current_env_key:
+                print(f"[run] {_key_preview(current_env_key, 'ENVIRONMENT_API_KEY (current)')}")
+        except Exception:
+            pass
+        if isinstance(js, dict):
+            detail = js.get("detail")
+            if isinstance(detail, dict):
+                try:
+                    sent_key = detail.get("sent_key")
+                    if isinstance(sent_key, str):
+                        print(f"[run] Backend detail.sent_key {_key_preview(sent_key, 'detail.sent_key')}")
+                except Exception:
+                    pass
+                try:
+                    sent_keys = detail.get("sent_keys")
+                    if isinstance(sent_keys, (list, tuple)):
+                        previews = []
+                        for idx, val in enumerate(sent_keys):
+                            if isinstance(val, str):
+                                previews.append(_key_preview(val, f"detail.sent_keys[{idx}]"))
+                        if previews:
+                            joined = "; ".join(previews)
+                            print(f"[run] Backend detail.sent_keys previews: {joined}")
+                except Exception:
+                    pass
+                try:
+                    key_prefix = detail.get("sent_key_prefix")
+                    if isinstance(key_prefix, str):
+                        print(f"[run] Backend detail.sent_key_prefix={key_prefix}")
+                except Exception:
+                    pass
+                try:
+                    health_url = detail.get("health_url")
+                    if isinstance(health_url, str):
+                        print(f"[run] Backend detail.health_url={health_url}")
+                except Exception:
+                    pass
         # Extra hints for auth failures
         try:
             sk = (env.synth_api_key or "").strip()

{synth_ai-0.2.8.dev9 → synth_ai-0.2.8.dev11}/synth_ai/demos/demo_task_apps/math/modal_task_app.py

@@ -358,122 +358,137 @@ def fastapi_app():
                 pass
             return []
 
-        # Build minimal context and execute ops
+        # Single-step rollout: one agent call followed by evaluation of the returned tool answer
         history: list[dict[str, Any]] = []
         steps: list[dict[str, Any]] = []
         total_reward = 0.0
-        last_llm_text: str | None = None
-        last_actions: list[str] = []
-        for op in ops or []:
-            if op == "agent":
-                user_prompt = (
-                    str(question)
-                    if isinstance(question, (str, int, float)) and str(question).strip()
-                    else "Solve the problem. Provide answer steps succinctly."
-                )
-                payload = {
-                    "model": model,
-                    "messages": [{"role": "user", "content": user_prompt}],
-                    "tools": [{
-                        "type": "function",
-                        "function": {"name": "interact", "parameters": {"type": "object", "properties": {"actions": {"type": "array", "items": {"type": "string"}}}, "required": ["actions"]}},
-                    }],
-                    "max_tokens": 256,
-                    "temperature": 0.2,
-                }
-                to_send = _prepare_payload(model if isinstance(model, str) else None, payload)
-                # Print prompts and tools exposed to the model
-                try:
-                    tool_names = []
-                    for t in (payload.get("tools") or []):
-                        if isinstance(t, dict):
-                            fn = (t.get("function") or {}) if isinstance(t.get("function"), dict) else {}
-                            name = fn.get("name")
-                            if isinstance(name, str):
-                                tool_names.append(name)
-                    print(f"[math] system: <none>", flush=True)
-                    print(f"[math] user: {user_prompt}", flush=True)
-                    print(f"[math] tools: {tool_names}", flush=True)
-                except Exception:
-                    pass
-                headers = {}
-                if "/proxy" in inference_url:
-                    sk = os.environ.get("SYNTH_API_KEY")
-                    if sk:
-                        headers["Authorization"] = f"Bearer {sk}"
-                with httpx.Client(timeout=httpx.Timeout(180.0), follow_redirects=True) as client:
-                    resp = client.post(f"{inference_url}/v1/chat/completions", json=to_send, headers=headers)
-                    try:
-                        data = resp.json()
-                    except Exception:
-                        data = {"error": "invalid_json", "raw": resp.text[:400]}
-
-                # Extract assistant text for visibility/correctness
-                llm_text = None
-                try:
-                    _choices = data.get("choices") if isinstance(data, dict) else None
-                    if isinstance(_choices, list) and _choices:
-                        _msg = _choices[0].get("message", {}) if isinstance(_choices[0], dict) else {}
-                        if isinstance(_msg, dict):
-                            _content = _msg.get("content")
-                            if isinstance(_content, str) and _content.strip():
-                                llm_text = _content
-                except Exception:
-                    llm_text = None
-
-                # Print question, model output, and correctness if we have an expected answer
-                try:
-                    if question is not None:
-                        print(f"[math] question: {question}", flush=True)
-                    if llm_text is not None:
-                        print(f"[math] llm: {llm_text}", flush=True)
-                    if expected_answer is not None and llm_text is not None:
-                        exp = str(expected_answer).strip()
-                        got = llm_text.strip()
-                        is_correct = exp and (exp in got)
-                        print(f"[math] correct: {bool(is_correct)} (expected fragment: {exp})", flush=True)
-                except Exception:
-                    pass
-                last_llm_text = llm_text
-                acts = _parse_tool_actions(data) or []
-                last_actions = acts if isinstance(acts, list) else []
-                steps.append({"obs": {}, "tool_calls": [{"tool_name": "interact", "arguments": _json.dumps({"actions": acts})}], "reward": None, "done": False, "truncated": False, "info": None})
-                history.append({"actions": acts})
-            elif op == "env":
-                # Compute a simple correctness-based reward if expected answer available
-                reward_val = 0.0
-                try:
-                    if expected_answer is not None:
-                        # Prefer explicit tool-call answer from last_actions
-                        candidate = ""
-                        if isinstance(last_actions, list) and last_actions:
-                            # Take the last non-empty action as the final answer
-                            for s in reversed(last_actions):
-                                if isinstance(s, str) and s.strip():
-                                    candidate = s.strip()
-                                    break
-                        # Fallback to parse from llm_text if tool actions absent
-                        if not candidate and last_llm_text is not None:
-                            candidate = _extract_boxed(last_llm_text) or last_llm_text
-                        exp_raw = _extract_boxed(str(expected_answer)) or str(expected_answer)
-                        got_raw = candidate
-                        exp_n = _normalize_answer_text(exp_raw)
-                        got_n = _normalize_answer_text(got_raw)
-                        if exp_n and exp_n in got_n:
-                            reward_val = 1.0
-                except Exception:
-                    reward_val = 0.0
-                steps.append({"obs": {}, "tool_calls": [], "reward": reward_val, "done": False, "truncated": False, "info": None})
-                total_reward += float(reward_val)
-            else:
-                continue
 
-        # Compose response similar to SDK contract (simplified)
+        user_prompt = (
+            str(question)
+            if isinstance(question, (str, int, float)) and str(question).strip()
+            else "Solve the problem. Provide answer steps succinctly."
+        )
+        payload = {
+            "model": model,
+            "messages": [{"role": "user", "content": user_prompt}],
+            "tools": [{
+                "type": "function",
+                "function": {"name": "interact", "parameters": {"type": "object", "properties": {"actions": {"type": "array", "items": {"type": "string"}}}, "required": ["actions"]}},
+            }],
+            "max_tokens": 256,
+            "temperature": 0.2,
+        }
+        to_send = _prepare_payload(model if isinstance(model, str) else None, payload)
+
+        try:
+            tool_names = []
+            for t in (payload.get("tools") or []):
+                if isinstance(t, dict):
+                    fn = (t.get("function") or {}) if isinstance(t.get("function"), dict) else {}
+                    name = fn.get("name")
+                    if isinstance(name, str):
+                        tool_names.append(name)
+            print(f"[math] system: <none>", flush=True)
+            print(f"[math] user: {user_prompt}", flush=True)
+            print(f"[math] tools: {tool_names}", flush=True)
+        except Exception:
+            pass
+
+        headers = {}
+        if "/proxy" in inference_url:
+            sk = os.environ.get("SYNTH_API_KEY")
+            if sk:
+                headers["Authorization"] = f"Bearer {sk}"
+        with httpx.Client(timeout=httpx.Timeout(180.0), follow_redirects=True) as client:
+            resp = client.post(f"{inference_url}/v1/chat/completions", json=to_send, headers=headers)
+            try:
+                data = resp.json()
+            except Exception:
+                data = {"error": "invalid_json", "raw": resp.text[:400]}
+
+        llm_text = None
+        try:
+            _choices = data.get("choices") if isinstance(data, dict) else None
+            if isinstance(_choices, list) and _choices:
+                _msg = _choices[0].get("message", {}) if isinstance(_choices[0], dict) else {}
+                if isinstance(_msg, dict):
+                    _content = _msg.get("content")
+                    if isinstance(_content, str) and _content.strip():
+                        llm_text = _content
+        except Exception:
+            llm_text = None
+
+        try:
+            if question is not None:
+                print(f"[math] question: {question}", flush=True)
+            if llm_text is not None:
+                print(f"[math] llm: {llm_text}", flush=True)
+            if expected_answer is not None and llm_text is not None:
+                exp = str(expected_answer).strip()
+                got = llm_text.strip()
+                is_correct = exp and (exp in got)
+                print(f"[math] correct: {bool(is_correct)} (expected fragment: {exp})", flush=True)
+        except Exception:
+            pass
+
+        tool_actions = _parse_tool_actions(data)
+        history.append({"actions": tool_actions})
+        steps.append({
+            "obs": {},
+            "tool_calls": [{"tool_name": "interact", "arguments": _json.dumps({"actions": tool_actions})}],
+            "reward": None,
+            "done": False,
+            "truncated": False,
+            "info": None,
+        })
+
+        # Evaluate answer correctness using tool output (or fall back to assistant text)
+        reward_val = 0.0
+        candidate = ""
+        try:
+            if isinstance(tool_actions, list):
+                for s in reversed(tool_actions):
+                    if isinstance(s, str) and s.strip():
+                        candidate = s.strip()
+                        break
+            if not candidate and llm_text is not None:
+                candidate = _extract_boxed(llm_text) or llm_text
+            if expected_answer is not None:
+                exp_raw = _extract_boxed(str(expected_answer)) or str(expected_answer)
+                got_raw = candidate
+                exp_n = _normalize_answer_text(exp_raw)
+                got_n = _normalize_answer_text(got_raw)
+                if exp_n and exp_n in got_n:
+                    reward_val = 1.0
+        except Exception:
+            reward_val = 0.0
+
+        total_reward += float(reward_val)
+        steps.append({
+            "obs": {},
+            "tool_calls": [],
+            "reward": reward_val,
+            "done": True,
+            "truncated": False,
+            "info": None,
+        })
+
         return {
             "run_id": run_id,
-            "trajectories": [{"env_id": env_name, "policy_id": (policy or {}).get("policy_name") or "math-react", "steps": steps, "final": {"observation": {}}, "length": len(steps)}],
+            "trajectories": [{
+                "env_id": env_name,
+                "policy_id": (policy or {}).get("policy_name") or "math-react",
+                "steps": steps,
+                "final": {"observation": {}},
+                "length": len(steps),
+            }],
             "branches": {},
-            "metrics": {"episode_returns": [total_reward], "mean_return": float(total_reward), "num_steps": len(steps), "num_episodes": 1},
+            "metrics": {
+                "episode_returns": [total_reward],
+                "mean_return": float(total_reward),
+                "num_steps": len(steps),
+                "num_episodes": 1,
+            },
             "aborted": False,
             "ops_executed": len(steps),
         }

{synth_ai-0.2.8.dev9 → synth_ai-0.2.8.dev11/synth_ai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.8.dev9
+Version: 0.2.8.dev11
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT