synth-ai 0.2.6.dev3__tar.gz → 0.2.6.dev4__tar.gz

{synth_ai-0.2.6.dev3/synth_ai.egg-info → synth_ai-0.2.6.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.6.dev3
+Version: 0.2.6.dev4
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT

{synth_ai-0.2.6.dev3 → synth_ai-0.2.6.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synth-ai"
-version = "0.2.6.dev3"
+version = "0.2.6.dev4"
 description = "RL as a service SDK - Core AI functionality and tracing"
 authors = [{name = "Synth AI", email = "josh@usesynth.ai"}]
 license = "MIT"

{synth_ai-0.2.6.dev3 → synth_ai-0.2.6.dev4}/synth_ai/__init__.py

@@ -25,7 +25,7 @@ tracing = None  # type: ignore
 EventPartitionElement = RewardSignal = SystemTrace = TrainingQuestion = None  # type: ignore
 trace_event_async = trace_event_sync = upload = None  # type: ignore
 
-__version__ = "0.2.6.dev1"
+__version__ = "0.2.6.dev4"
 __all__ = [
     "LM",
     "OpenAI",

{synth_ai-0.2.6.dev3 → synth_ai-0.2.6.dev4}/synth_ai/demos/core/cli.py

@@ -938,96 +938,6 @@ def cmd_run(args: argparse.Namespace) -> int:
     return 0
 
 
-def cmd_eval(args: argparse.Namespace) -> int:
-    env = demo_core.load_env()
-    # Ensure required env
-    if not env.task_app_base_url:
-        print("Task app URL missing. Run: uvx synth-ai rl_demo deploy")
-        return 1
-    # Load config: prefer CWD demo_config.toml; else packaged default
-    cfg_path: str | None = None
-    if getattr(args, "config", None):
-        p = os.path.abspath(args.config)
-        if not os.path.isfile(p):
-            print(f"Config not found: {p}")
-            return 1
-        cfg_path = p
-    else:
-        cwd_prepared = os.path.abspath(os.path.join(os.getcwd(), "demo_config.toml"))
-        if os.path.isfile(cwd_prepared):
-            cfg_path = cwd_prepared
-        else:
-            packaged = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "demo_task_apps", "math", "config.toml"))
-            cfg_path = packaged if os.path.isfile(packaged) else None
-    if not cfg_path:
-        print("No config TOML found.")
-        return 1
-
-    import tomllib
-    with open(cfg_path, "rb") as fh:
-        inline_cfg = tomllib.load(fh)
-
-    # Model selection prompt
-    default_model = (args.model or (inline_cfg.get("model", {}) or {}).get("name") or "Qwen/Qwen3-0.6B")
-    entered = input(f"Model to evaluate [{default_model}]: ").strip()
-    model = entered or default_model
-    confirm = (input(f"Use model '{model}'? [Y/n]: ").strip().lower() or "y").startswith("y")
-    if not confirm:
-        print("Aborted by user.")
-        return 1
-
-    # Build on-board rollout request to the Task App (no backend RL job)
-    # Use Synth backend chat-completions proxy as inference URL (derive from DEV_BACKEND_URL)
-    # Ensure /api suffix for backend, then use proxy prefix for chat completions
-    backend_api = (env.dev_backend_url or "https://agent-learning.onrender.com/api").rstrip("/")
-    if not backend_api.endswith("/api"):
-        backend_api = f"{backend_api}/api"
-    inference_url = f"{backend_api}/proxy"
-    # ops: alternate agent/env for a small number of decisions (from config max_steps_per_episode if present)
-    try:
-        steps = int((inline_cfg.get("rollout", {}) or {}).get("max_steps_per_episode", 4))
-    except Exception:
-        steps = 4
-    ops: list[str] = []
-    for _ in range(max(1, steps // 2)):
-        ops.extend(["agent", "env"])
-    env_name = (inline_cfg.get("rollout", {}) or {}).get("env_name") or "math"
-    policy_name = (inline_cfg.get("rollout", {}) or {}).get("policy_name") or "math-react"
-    run_id = f"eval-{int(time.time())}"
-    body: Dict[str, Any] = {
-        "run_id": run_id,
-        "env": {
-            "env_name": env_name,
-            "config": inline_cfg.get("rollout", {}) or {},
-        },
-        "policy": {
-            "policy_name": policy_name,
-            "config": {"model": model, "inference_url": inference_url},
-        },
-        "ops": ops,
-        "on_done": "terminate",
-    }
-    # POST to task app rollout endpoint
-    headers = {"Content-Type": "application/json"}
-    if env.env_api_key:
-        headers["X-API-Key"] = env.env_api_key
-    rc, resp = _http("POST", env.task_app_base_url.rstrip("/") + "/rollout", headers=headers, body=body)
-    if rc not in (200, 201) or not isinstance(resp, dict):
-        print("Eval rollout failed:", rc)
-        try:
-            print(json.dumps(resp, indent=2) if isinstance(resp, dict) else str(resp))
-        except Exception:
-            print(str(resp))
-        print("Request body was:\n" + json.dumps(body, indent=2))
-        return 2
-    metrics = (resp.get("metrics") if isinstance(resp, dict) else None) or {}
-    mean = metrics.get("mean_return")
-    if mean is not None:
-        print(f"eval.reward_mean={mean}")
-    else:
-        print(json.dumps(resp, indent=2))
-    return 0
-
 def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser(prog="synth-ai")
     sub = p.add_subparsers(dest="cmd")
@@ -1069,14 +979,6 @@ def main(argv: list[str] | None = None) -> int:
 
     _add_parser(["rl_demo.run", "demo.run"], configure=_run_opts)
 
-    def _eval_opts(parser):
-        parser.add_argument("--config", type=str, default=None, help="Path to TOML config (optional)")
-        parser.add_argument("--model", type=str, default=None, help="Model to evaluate (default Qwen/Qwen3-0.6B)")
-        parser.add_argument("--timeout", type=int, default=300, help="Seconds to wait for metrics")
-        parser.set_defaults(func=cmd_eval)
-
-    _add_parser(["rl_demo.eval", "demo.eval"], configure=_eval_opts)
-
     args = p.parse_args(argv)
     if not hasattr(args, "func"):
         p.print_help()

{synth_ai-0.2.6.dev3 → synth_ai-0.2.6.dev4/synth_ai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.6.dev3
+Version: 0.2.6.dev4
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT