synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synth-ai might be problematic. Click here for more details.

Files changed (169) hide show
  1. examples/baseline/banking77_baseline.py +204 -0
  2. examples/baseline/crafter_baseline.py +407 -0
  3. examples/baseline/pokemon_red_baseline.py +326 -0
  4. examples/baseline/simple_baseline.py +56 -0
  5. examples/baseline/warming_up_to_rl_baseline.py +239 -0
  6. examples/blog_posts/gepa/README.md +355 -0
  7. examples/blog_posts/gepa/configs/banking77_gepa_local.toml +95 -0
  8. examples/blog_posts/gepa/configs/banking77_gepa_test.toml +82 -0
  9. examples/blog_posts/gepa/configs/banking77_mipro_local.toml +52 -0
  10. examples/blog_posts/gepa/configs/hotpotqa_gepa_local.toml +59 -0
  11. examples/blog_posts/gepa/configs/hotpotqa_gepa_qwen.toml +36 -0
  12. examples/blog_posts/gepa/configs/hotpotqa_mipro_local.toml +53 -0
  13. examples/blog_posts/gepa/configs/hover_gepa_local.toml +59 -0
  14. examples/blog_posts/gepa/configs/hover_gepa_qwen.toml +36 -0
  15. examples/blog_posts/gepa/configs/hover_mipro_local.toml +53 -0
  16. examples/blog_posts/gepa/configs/ifbench_gepa_local.toml +59 -0
  17. examples/blog_posts/gepa/configs/ifbench_gepa_qwen.toml +36 -0
  18. examples/blog_posts/gepa/configs/ifbench_mipro_local.toml +53 -0
  19. examples/blog_posts/gepa/configs/pupa_gepa_local.toml +60 -0
  20. examples/blog_posts/gepa/configs/pupa_mipro_local.toml +54 -0
  21. examples/blog_posts/gepa/deploy_banking77_task_app.sh +41 -0
  22. examples/blog_posts/gepa/gepa_baseline.py +204 -0
  23. examples/blog_posts/gepa/query_prompts_example.py +97 -0
  24. examples/blog_posts/gepa/run_gepa_banking77.sh +87 -0
  25. examples/blog_posts/gepa/task_apps.py +105 -0
  26. examples/blog_posts/gepa/test_gepa_local.sh +67 -0
  27. examples/blog_posts/gepa/verify_banking77_setup.sh +123 -0
  28. examples/blog_posts/pokemon_vl/configs/eval_gpt5nano.toml +26 -0
  29. examples/blog_posts/pokemon_vl/configs/eval_qwen3_vl.toml +12 -10
  30. examples/blog_posts/pokemon_vl/configs/train_rl_from_sft.toml +1 -0
  31. examples/blog_posts/pokemon_vl/extract_images.py +239 -0
  32. examples/blog_posts/pokemon_vl/pokemon_vl_baseline.py +326 -0
  33. examples/blog_posts/pokemon_vl/run_eval_extract_images.py +209 -0
  34. examples/blog_posts/pokemon_vl/run_qwen_eval_extract_images.py +212 -0
  35. examples/blog_posts/pokemon_vl/text_box_analysis.md +106 -0
  36. examples/blog_posts/warming_up_to_rl/ARCHITECTURE.md +195 -0
  37. examples/blog_posts/warming_up_to_rl/FINAL_TEST_RESULTS.md +127 -0
  38. examples/blog_posts/warming_up_to_rl/INFERENCE_SUCCESS.md +132 -0
  39. examples/blog_posts/warming_up_to_rl/SMOKE_TESTING.md +164 -0
  40. examples/blog_posts/warming_up_to_rl/SMOKE_TEST_COMPLETE.md +253 -0
  41. examples/blog_posts/warming_up_to_rl/configs/eval_baseline_qwen32b_10x20.toml +25 -0
  42. examples/blog_posts/warming_up_to_rl/configs/eval_ft_qwen4b_10x20.toml +26 -0
  43. examples/blog_posts/warming_up_to_rl/configs/filter_high_reward_dataset.toml +1 -1
  44. examples/blog_posts/warming_up_to_rl/configs/smoke_test.toml +75 -0
  45. examples/blog_posts/warming_up_to_rl/configs/train_rl_from_sft.toml +60 -10
  46. examples/blog_posts/warming_up_to_rl/configs/train_sft_qwen4b.toml +1 -1
  47. examples/blog_posts/warming_up_to_rl/warming_up_to_rl_baseline.py +187 -0
  48. examples/multi_step/configs/VERILOG_REWARDS.md +4 -0
  49. examples/multi_step/configs/VERILOG_RL_CHECKLIST.md +4 -0
  50. examples/multi_step/configs/crafter_rl_outcome.toml +1 -0
  51. examples/multi_step/configs/crafter_rl_stepwise_shaped.toml +1 -0
  52. examples/multi_step/configs/crafter_rl_stepwise_simple.toml +1 -0
  53. examples/rl/configs/rl_from_base_qwen17.toml +1 -0
  54. examples/swe/task_app/hosted/inference/openai_client.py +0 -34
  55. examples/swe/task_app/hosted/policy_routes.py +17 -0
  56. examples/swe/task_app/hosted/rollout.py +4 -2
  57. examples/task_apps/banking77/__init__.py +6 -0
  58. examples/task_apps/banking77/banking77_task_app.py +841 -0
  59. examples/task_apps/banking77/deploy_wrapper.py +46 -0
  60. examples/task_apps/crafter/CREATE_SFT_DATASET.md +4 -0
  61. examples/task_apps/crafter/FILTER_COMMAND_STATUS.md +4 -0
  62. examples/task_apps/crafter/FILTER_COMMAND_SUCCESS.md +4 -0
  63. examples/task_apps/crafter/task_app/grpo_crafter.py +24 -2
  64. examples/task_apps/crafter/task_app/synth_envs_hosted/hosted_app.py +49 -0
  65. examples/task_apps/crafter/task_app/synth_envs_hosted/inference/openai_client.py +355 -58
  66. examples/task_apps/crafter/task_app/synth_envs_hosted/policy_routes.py +68 -7
  67. examples/task_apps/crafter/task_app/synth_envs_hosted/rollout.py +78 -21
  68. examples/task_apps/crafter/task_app/synth_envs_hosted/utils.py +194 -1
  69. examples/task_apps/gepa_benchmarks/__init__.py +7 -0
  70. examples/task_apps/gepa_benchmarks/common.py +260 -0
  71. examples/task_apps/gepa_benchmarks/hotpotqa_task_app.py +507 -0
  72. examples/task_apps/gepa_benchmarks/hover_task_app.py +436 -0
  73. examples/task_apps/gepa_benchmarks/ifbench_task_app.py +563 -0
  74. examples/task_apps/gepa_benchmarks/pupa_task_app.py +460 -0
  75. examples/task_apps/pokemon_red/README_IMAGE_ONLY_EVAL.md +4 -0
  76. examples/task_apps/pokemon_red/task_app.py +254 -36
  77. examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml +1 -0
  78. examples/warming_up_to_rl/task_app/grpo_crafter.py +53 -4
  79. examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py +49 -0
  80. examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py +152 -41
  81. examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py +31 -1
  82. examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py +33 -3
  83. examples/warming_up_to_rl/task_app/synth_envs_hosted/utils.py +67 -0
  84. examples/workflows/math_rl/configs/rl_from_base_qwen17.toml +1 -0
  85. synth_ai/api/train/builders.py +90 -1
  86. synth_ai/api/train/cli.py +396 -21
  87. synth_ai/api/train/config_finder.py +13 -2
  88. synth_ai/api/train/configs/__init__.py +15 -1
  89. synth_ai/api/train/configs/prompt_learning.py +442 -0
  90. synth_ai/api/train/configs/rl.py +29 -0
  91. synth_ai/api/train/task_app.py +1 -1
  92. synth_ai/api/train/validators.py +277 -0
  93. synth_ai/baseline/__init__.py +25 -0
  94. synth_ai/baseline/config.py +209 -0
  95. synth_ai/baseline/discovery.py +214 -0
  96. synth_ai/baseline/execution.py +146 -0
  97. synth_ai/cli/__init__.py +85 -17
  98. synth_ai/cli/__main__.py +0 -0
  99. synth_ai/cli/claude.py +70 -0
  100. synth_ai/cli/codex.py +84 -0
  101. synth_ai/cli/commands/__init__.py +1 -0
  102. synth_ai/cli/commands/baseline/__init__.py +12 -0
  103. synth_ai/cli/commands/baseline/core.py +637 -0
  104. synth_ai/cli/commands/baseline/list.py +93 -0
  105. synth_ai/cli/commands/eval/core.py +13 -10
  106. synth_ai/cli/commands/filter/core.py +53 -17
  107. synth_ai/cli/commands/help/core.py +0 -1
  108. synth_ai/cli/commands/smoke/__init__.py +7 -0
  109. synth_ai/cli/commands/smoke/core.py +1436 -0
  110. synth_ai/cli/commands/status/subcommands/pricing.py +22 -0
  111. synth_ai/cli/commands/status/subcommands/usage.py +203 -0
  112. synth_ai/cli/commands/train/judge_schemas.py +1 -0
  113. synth_ai/cli/commands/train/judge_validation.py +1 -0
  114. synth_ai/cli/commands/train/validation.py +0 -57
  115. synth_ai/cli/demo.py +35 -3
  116. synth_ai/cli/deploy/__init__.py +40 -25
  117. synth_ai/cli/deploy.py +162 -0
  118. synth_ai/cli/legacy_root_backup.py +14 -8
  119. synth_ai/cli/opencode.py +107 -0
  120. synth_ai/cli/root.py +9 -5
  121. synth_ai/cli/task_app_deploy.py +1 -1
  122. synth_ai/cli/task_apps.py +53 -53
  123. synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py +7 -4
  124. synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py +9 -5
  125. synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py +4 -3
  126. synth_ai/judge_schemas.py +1 -0
  127. synth_ai/learning/__init__.py +10 -0
  128. synth_ai/learning/prompt_learning_client.py +276 -0
  129. synth_ai/learning/prompt_learning_types.py +184 -0
  130. synth_ai/pricing/__init__.py +2 -0
  131. synth_ai/pricing/model_pricing.py +57 -0
  132. synth_ai/streaming/handlers.py +53 -4
  133. synth_ai/streaming/streamer.py +19 -0
  134. synth_ai/task/apps/__init__.py +1 -0
  135. synth_ai/task/config.py +2 -0
  136. synth_ai/task/tracing_utils.py +25 -25
  137. synth_ai/task/validators.py +44 -8
  138. synth_ai/task_app_cfgs.py +21 -0
  139. synth_ai/tracing_v3/config.py +162 -19
  140. synth_ai/tracing_v3/constants.py +1 -1
  141. synth_ai/tracing_v3/db_config.py +24 -38
  142. synth_ai/tracing_v3/storage/config.py +47 -13
  143. synth_ai/tracing_v3/storage/factory.py +3 -3
  144. synth_ai/tracing_v3/turso/daemon.py +113 -11
  145. synth_ai/tracing_v3/turso/native_manager.py +92 -16
  146. synth_ai/types.py +8 -0
  147. synth_ai/urls.py +11 -0
  148. synth_ai/utils/__init__.py +30 -1
  149. synth_ai/utils/agents.py +74 -0
  150. synth_ai/utils/bin.py +39 -0
  151. synth_ai/utils/cli.py +149 -5
  152. synth_ai/utils/env.py +17 -17
  153. synth_ai/utils/json.py +72 -0
  154. synth_ai/utils/modal.py +283 -1
  155. synth_ai/utils/paths.py +48 -0
  156. synth_ai/utils/uvicorn.py +113 -0
  157. {synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/METADATA +102 -4
  158. {synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/RECORD +162 -88
  159. synth_ai/cli/commands/deploy/__init__.py +0 -23
  160. synth_ai/cli/commands/deploy/core.py +0 -614
  161. synth_ai/cli/commands/deploy/errors.py +0 -72
  162. synth_ai/cli/commands/deploy/validation.py +0 -11
  163. synth_ai/cli/deploy/core.py +0 -5
  164. synth_ai/cli/deploy/errors.py +0 -23
  165. synth_ai/cli/deploy/validation.py +0 -5
  166. {synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/WHEEL +0 -0
  167. {synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/entry_points.txt +0 -0
  168. {synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/licenses/LICENSE +0 -0
  169. {synth_ai-0.2.17.dist-info → synth_ai-0.2.19.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,22 @@
1
+ from __future__ import annotations
2
+
3
+ import click
4
+ from rich.table import Table
5
+ from synth_ai.pricing.model_pricing import MODEL_PRICES
6
+
7
+ from ..formatters import console
8
+
9
+
10
+ @click.command("pricing", help="List supported provider/model rates (SDK static table).")
11
+ def pricing_command() -> None:
12
+ table = Table(title="Supported Models and Rates (USD/token)")
13
+ table.add_column("Provider", style="cyan", no_wrap=True)
14
+ table.add_column("Model", style="magenta")
15
+ table.add_column("Input USD", justify="right")
16
+ table.add_column("Output USD", justify="right")
17
+ for provider, models in MODEL_PRICES.items():
18
+ for model, rates in models.items():
19
+ table.add_row(provider, model, f"{rates.input_usd:.9f}", f"{rates.output_usd:.9f}")
20
+ console.print(table)
21
+
22
+
@@ -0,0 +1,203 @@
1
+ from __future__ import annotations
2
+
3
+ import contextlib
4
+ from datetime import UTC, datetime, timedelta
5
+ from typing import Any
6
+
7
+ import click
8
+
9
+ from ..client import StatusAPIClient
10
+ from ..errors import StatusAPIError
11
+ from ..formatters import console
12
+ from ..utils import common_options, resolve_context_config
13
+
14
+
15
+ def _parse_iso(ts: str | None) -> datetime | None:
16
+ if not ts:
17
+ return None
18
+ try:
19
+ # Python 3.11 handles 'YYYY-mm-ddTHH:MM:SS.ssssss+00:00' and '...Z'
20
+ if ts.endswith("Z"):
21
+ ts = ts.replace("Z", "+00:00")
22
+ return datetime.fromisoformat(ts)
23
+ except Exception:
24
+ return None
25
+
26
+
27
+ def _extract_total_usd(events: list[dict[str, Any]]) -> tuple[float, int]:
28
+ """Return (usd_total, tokens_total) for an arbitrary job's events.
29
+
30
+ Strategy:
31
+ - Prefer a consolidated total from any *.completed event with total_usd
32
+ - Next, prefer any *.billing.end event with total_usd
33
+ - Otherwise, combine usage.recorded's usd_tokens with billing.sandboxes' usd
34
+ and sum token counts if present
35
+ Works for prompt learning and other job types that follow similar conventions.
36
+ """
37
+ total_usd = 0.0
38
+ token_count = 0
39
+
40
+ # Prefer consolidated totals from completion events (any namespace)
41
+ for e in reversed(events):
42
+ typ = str(e.get("type") or "").lower()
43
+ if typ.endswith(".completed"):
44
+ data = e.get("data") or {}
45
+ try:
46
+ total_usd = float(data.get("total_usd") or 0.0)
47
+ except Exception:
48
+ total_usd = 0.0
49
+ # Try common token fields
50
+ tc = 0
51
+ for k in ("token_count_total", "token_count"):
52
+ try:
53
+ tc = int(data.get(k) or 0)
54
+ if tc:
55
+ break
56
+ except Exception:
57
+ pass
58
+ if not tc:
59
+ try:
60
+ tc = int((data.get("token_count_rollouts") or 0) + (data.get("token_count_mutation") or 0))
61
+ except Exception:
62
+ tc = 0
63
+ token_count = tc
64
+ return total_usd, token_count
65
+
66
+ # Next, billing.end if present with total_usd
67
+ for e in reversed(events):
68
+ typ = str(e.get("type") or "").lower()
69
+ if typ.endswith("billing.end"):
70
+ data = e.get("data") or {}
71
+ try:
72
+ total_usd = float(data.get("total_usd") or 0.0)
73
+ except Exception:
74
+ total_usd = 0.0
75
+ # token_count may not be present here; fall through to usage tokens calc
76
+ break
77
+
78
+ # Fallback: combine usage + sandboxes (prompt learning style); generic scan
79
+ usd_tokens = 0.0
80
+ sandbox_usd = 0.0
81
+ # token fields observed across tasks
82
+ token_fields = ("token_count_total", "token_count", "tokens_in", "tokens_out",
83
+ "token_count_rollouts", "token_count_mutation")
84
+ for e in events:
85
+ typ = str(e.get("type") or "").lower()
86
+ data = e.get("data") or {}
87
+ # generic usage-style aggregation
88
+ if "usage" in typ or typ.endswith("usage.recorded"):
89
+ with contextlib.suppress(Exception):
90
+ usd_tokens = float(data.get("usd_tokens") or data.get("usd_estimate") or 0.0)
91
+ # accumulate tokens if any
92
+ for k in token_fields:
93
+ with contextlib.suppress(Exception):
94
+ token_count += int(data.get(k) or 0)
95
+ # sandbox billing
96
+ if typ.endswith("billing.sandboxes"):
97
+ with contextlib.suppress(Exception):
98
+ sandbox_usd += float(data.get("usd") or 0.0)
99
+ return (total_usd or (usd_tokens + sandbox_usd)), token_count
100
+
101
+
102
+ @click.command("usage", help="Show recent usage (daily/weekly/monthly) and remaining budget if provided.")
103
+ @common_options()
104
+ @click.option("--budget-usd", type=float, default=None, help="Optional credit/budget to compute remaining.")
105
+ @click.option("--json", "output_json", is_flag=True, help="Emit machine-readable JSON.")
106
+ @click.pass_context
107
+ def usage_command(
108
+ ctx: click.Context,
109
+ base_url: str | None,
110
+ api_key: str | None,
111
+ timeout: float,
112
+ budget_usd: float | None,
113
+ output_json: bool,
114
+ ) -> None:
115
+ cfg = resolve_context_config(ctx, base_url=base_url, api_key=api_key, timeout=timeout)
116
+ now = datetime.now(UTC)
117
+ daily_cutoff = (now - timedelta(days=1)).isoformat()
118
+ weekly_cutoff = (now - timedelta(days=7)).isoformat()
119
+ monthly_cutoff = (now - timedelta(days=30)).isoformat()
120
+
121
+ async def _run() -> tuple[dict[str, float | int], dict[str, float | int], dict[str, float | int]]:
122
+ daily = {"usd": 0.0, "tokens": 0, "sandbox_seconds": 0.0}
123
+ weekly = {"usd": 0.0, "tokens": 0, "sandbox_seconds": 0.0}
124
+ monthly = {"usd": 0.0, "tokens": 0, "sandbox_seconds": 0.0}
125
+ async with StatusAPIClient(cfg) as client:
126
+ try:
127
+ jobs = await client.list_jobs(created_after=weekly_cutoff)
128
+ except StatusAPIError as exc:
129
+ raise click.ClickException(f"Backend error: {exc}") from exc
130
+ for j in jobs or []:
131
+ job_id = str(j.get("job_id") or j.get("id") or "")
132
+ if not job_id:
133
+ continue
134
+ try:
135
+ events = await client.get_job_events(job_id, since=weekly_cutoff)
136
+ except StatusAPIError:
137
+ events = []
138
+ if not events:
139
+ continue
140
+ # Use event timestamps for windowing
141
+ # Weekly
142
+ weekly_ev = [e for e in events if (_parse_iso(e.get("created_at")) or now) >= datetime.fromisoformat(weekly_cutoff)]
143
+ w_usd, w_tok = _extract_total_usd(weekly_ev)
144
+ weekly["usd"] += w_usd
145
+ weekly["tokens"] += w_tok
146
+ # sandbox seconds
147
+ for e in weekly_ev:
148
+ if str(e.get("type") or "").lower().endswith("billing.sandboxes"):
149
+ with contextlib.suppress(Exception):
150
+ weekly["sandbox_seconds"] += float((e.get("data") or {}).get("seconds") or 0.0)
151
+ # Daily
152
+ daily_ev = [e for e in events if (_parse_iso(e.get("created_at")) or now) >= datetime.fromisoformat(daily_cutoff)]
153
+ d_usd, d_tok = _extract_total_usd(daily_ev)
154
+ daily["usd"] += d_usd
155
+ daily["tokens"] += d_tok
156
+ for e in daily_ev:
157
+ if str(e.get("type") or "").lower().endswith("billing.sandboxes"):
158
+ with contextlib.suppress(Exception):
159
+ daily["sandbox_seconds"] += float((e.get("data") or {}).get("seconds") or 0.0)
160
+ # Monthly
161
+ monthly_ev = [e for e in events if (_parse_iso(e.get("created_at")) or now) >= datetime.fromisoformat(monthly_cutoff)]
162
+ m_usd, m_tok = _extract_total_usd(monthly_ev)
163
+ monthly["usd"] += m_usd
164
+ monthly["tokens"] += m_tok
165
+ for e in monthly_ev:
166
+ if str(e.get("type") or "").lower().endswith("billing.sandboxes"):
167
+ with contextlib.suppress(Exception):
168
+ monthly["sandbox_seconds"] += float((e.get("data") or {}).get("seconds") or 0.0)
169
+ return daily, weekly, monthly
170
+
171
+ daily, weekly, monthly = __import__("asyncio").run(_run())
172
+
173
+ if output_json:
174
+ import json as _json
175
+ payload: dict[str, Any] = {
176
+ "daily": {
177
+ "usd": round(float(daily["usd"]), 4),
178
+ "tokens": int(daily["tokens"]),
179
+ "sandbox_hours": round(float(daily["sandbox_seconds"]) / 3600.0, 4),
180
+ },
181
+ "weekly": {
182
+ "usd": round(float(weekly["usd"]), 4),
183
+ "tokens": int(weekly["tokens"]),
184
+ "sandbox_hours": round(float(weekly["sandbox_seconds"]) / 3600.0, 4),
185
+ },
186
+ "monthly": {
187
+ "usd": round(float(monthly["usd"]), 4),
188
+ "tokens": int(monthly["tokens"]),
189
+ "sandbox_hours": round(float(monthly["sandbox_seconds"]) / 3600.0, 4),
190
+ },
191
+ }
192
+ if budget_usd is not None:
193
+ payload["remaining_vs_budget"] = round(max(0.0, float(budget_usd) - float(weekly["usd"])), 4)
194
+ console.print(_json.dumps(payload))
195
+ return
196
+
197
+ console.print(f"Daily usage: ${float(daily['usd']):.2f} | tokens {int(daily['tokens'])} | sandbox {float(daily['sandbox_seconds'])/3600.0:.2f}h")
198
+ console.print(f"Weekly usage: ${float(weekly['usd']):.2f} | tokens {int(weekly['tokens'])} | sandbox {float(weekly['sandbox_seconds'])/3600.0:.2f}h")
199
+ console.print(f"Monthly usage: ${float(monthly['usd']):.2f} | tokens {int(monthly['tokens'])} | sandbox {float(monthly['sandbox_seconds'])/3600.0:.2f}h")
200
+ if budget_usd is not None:
201
+ remaining = max(0.0, float(budget_usd) - float(weekly["usd"]))
202
+ console.print(f"Remaining (vs weekly budget ${float(budget_usd):.2f}): ${remaining:.2f}")
203
+
@@ -197,3 +197,4 @@ def build_judge_http_options(
197
197
  payload["rubric_overrides"] = options_config.rubric_overrides
198
198
 
199
199
  return payload
200
+
@@ -302,3 +302,4 @@ def check_for_deprecated_fields(toml_config: MutableMapping[str, Any]) -> dict[s
302
302
  deprecated["judge.options"] = options_found
303
303
 
304
304
  return deprecated
305
+
@@ -261,23 +261,6 @@ def validate_rl_config(config: MutableMapping[str, Any]) -> dict[str, Any]:
261
261
  hint="Specify gpus_for_vllm, gpus_for_training, etc."
262
262
  )
263
263
 
264
- # Validate vllm section and tensor_parallel consistency
265
- vllm = config.get("vllm", {})
266
- topology_tensor_parallel = topology.get("tensor_parallel")
267
- vllm_tensor_parallel = vllm.get("tensor_parallel_size")
268
-
269
- if topology_tensor_parallel and not vllm_tensor_parallel:
270
- raise InvalidRLConfigError(
271
- detail="Both [topology].tensor_parallel and [vllm].tensor_parallel_size must be provided",
272
- hint=f"Add [vllm] section with tensor_parallel_size={topology_tensor_parallel}"
273
- )
274
-
275
- if vllm_tensor_parallel and not topology_tensor_parallel:
276
- raise InvalidRLConfigError(
277
- detail="Both [topology].tensor_parallel and [vllm].tensor_parallel_size must be provided",
278
- hint=f"Add tensor_parallel={vllm_tensor_parallel} to [topology] section"
279
- )
280
-
281
264
  # Check for training section and its required fields
282
265
  training = config.get("training", {})
283
266
  if training:
@@ -288,8 +271,6 @@ def validate_rl_config(config: MutableMapping[str, Any]) -> dict[str, Any]:
288
271
  "batch_size": "batch size",
289
272
  "group_size": "group size",
290
273
  "learning_rate": "learning rate",
291
- "weight_sync_interval": "weight sync interval",
292
- "log_interval": "logging interval",
293
274
  }
294
275
 
295
276
  for field, description in required_training_fields.items():
@@ -298,44 +279,6 @@ def validate_rl_config(config: MutableMapping[str, Any]) -> dict[str, Any]:
298
279
  detail=f"[training].{field} is required ({description})",
299
280
  hint=f"Add {field} to the [training] section"
300
281
  )
301
-
302
- # Validate weight_sync_interval is positive
303
- weight_sync_interval = training.get("weight_sync_interval")
304
- if weight_sync_interval is not None and weight_sync_interval <= 0:
305
- raise InvalidRLConfigError(
306
- detail="[training].weight_sync_interval must be a positive integer",
307
- hint="Set weight_sync_interval to a value >= 1"
308
- )
309
-
310
- # Ensure weight_sync block exists with proper defaults
311
- # Backend requires mode="direct" - always inject it
312
- if "weight_sync" not in training:
313
- training["weight_sync"] = {
314
- "enable": True,
315
- "mode": "direct", # Backend requirement
316
- "targets": ["policy"],
317
- "interval": training.get("weight_sync_interval", 1),
318
- }
319
- else:
320
- weight_sync = training["weight_sync"]
321
- # Always force mode to "direct" (backend requirement)
322
- weight_sync["mode"] = "direct"
323
-
324
- # Validate existing weight_sync block
325
- if not weight_sync.get("enable"):
326
- raise InvalidRLConfigError(
327
- detail="[training.weight_sync].enable must be true",
328
- hint="Set enable=true in the weight_sync section"
329
- )
330
- targets = weight_sync.get("targets", [])
331
- if not targets or "policy" not in targets:
332
- raise InvalidRLConfigError(
333
- detail="[training.weight_sync].targets must include 'policy'",
334
- hint="Add targets=['policy'] to the weight_sync section"
335
- )
336
- # Inject interval if not present
337
- if "interval" not in weight_sync:
338
- weight_sync["interval"] = training.get("weight_sync_interval", 1)
339
282
 
340
283
  # Check for evaluation section
341
284
  evaluation = config.get("evaluation", {})
synth_ai/cli/demo.py CHANGED
@@ -1,5 +1,37 @@
1
- from __future__ import annotations
1
+ import shutil
2
+ from pathlib import Path
2
3
 
3
- from synth_ai.cli.commands.demo.core import register
4
+ import click
4
5
 
5
- __all__ = ["register"]
6
+
7
+ DEMO_SOURCES: dict[str, str] = {
8
+ "local": "crafter",
9
+ "modal": "math"
10
+ }
11
+
12
+
13
+ @click.command()
14
+ @click.option(
15
+ "--runtime",
16
+ "runtime",
17
+ type=click.Choice(tuple(DEMO_SOURCES.keys()), case_sensitive=False),
18
+ default="local",
19
+ show_default=True,
20
+ help="Select runtime to load a demo task app to your cwd. Options: local, modal"
21
+ )
22
+ def demo_cmd(runtime: str) -> None:
23
+ runtime_key = runtime.lower()
24
+ demo_name = DEMO_SOURCES[runtime_key]
25
+ package_root = Path(__file__).resolve().parents[1]
26
+ src = package_root / "demos" / demo_name
27
+ if not src.exists():
28
+ raise click.ClickException(f"Demo source directory not found: {src}")
29
+
30
+ dst = Path.cwd() / src.name
31
+ if dst.exists():
32
+ raise click.ClickException(
33
+ f"Destination already exists: {dst}. Remove it first if you want to re-copy."
34
+ )
35
+
36
+ shutil.copytree(src, dst)
37
+ click.echo(f"Copied {demo_name} demo to {dst}")
@@ -1,28 +1,43 @@
1
+ """Deploy command package - imports from deploy.py module."""
1
2
  from __future__ import annotations
2
3
 
3
- from .core import command, get_command
4
- from .errors import (
5
- DeployCliError,
6
- EnvFileDiscoveryError,
7
- EnvironmentKeyLoadError,
8
- EnvKeyPreflightError,
9
- MissingEnvironmentApiKeyError,
10
- ModalCliResolutionError,
11
- ModalExecutionError,
12
- TaskAppNotFoundError,
13
- )
14
- from .validation import validate_deploy_options
4
+ # Import from the deploy.py module file (using importlib to avoid conflicts)
5
+ # This package exists for backwards compatibility
6
+ import importlib
7
+ import importlib.util
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import TYPE_CHECKING
15
11
 
16
- __all__ = [
17
- "command",
18
- "get_command",
19
- "DeployCliError",
20
- "MissingEnvironmentApiKeyError",
21
- "EnvironmentKeyLoadError",
22
- "EnvFileDiscoveryError",
23
- "TaskAppNotFoundError",
24
- "ModalCliResolutionError",
25
- "ModalExecutionError",
26
- "EnvKeyPreflightError",
27
- "validate_deploy_options",
28
- ]
12
+ if TYPE_CHECKING:
13
+ from click import Command
14
+
15
+ try:
16
+ # Import the deploy.py module directly by file path to avoid package/module conflict
17
+ deploy_module_path = Path(__file__).parent.parent / "deploy.py"
18
+ if deploy_module_path.exists():
19
+ spec = importlib.util.spec_from_file_location("synth_ai.cli.deploy_module", deploy_module_path)
20
+ if spec and spec.loader:
21
+ deploy_module = importlib.util.module_from_spec(spec)
22
+ sys.modules["synth_ai.cli.deploy_module"] = deploy_module
23
+ spec.loader.exec_module(deploy_module)
24
+ command: Command | None = getattr(deploy_module, "deploy_cmd", None) # type: ignore[assignment]
25
+ deploy_cmd: Command | None = command # type: ignore[assignment]
26
+ else:
27
+ raise ImportError("Could not load deploy.py")
28
+ else:
29
+ raise ImportError("deploy.py not found")
30
+
31
+ get_command: None = None # Not used in current implementation
32
+
33
+ __all__: list[str] = [
34
+ "command",
35
+ "deploy_cmd",
36
+ ]
37
+ except Exception:
38
+ # If deploy.py doesn't exist or fails to import, provide a stub
39
+ command: Command | None = None # type: ignore[assignment]
40
+ deploy_cmd: Command | None = None # type: ignore[assignment]
41
+ get_command: None = None
42
+
43
+ __all__: list[str] = []
synth_ai/cli/deploy.py ADDED
@@ -0,0 +1,162 @@
1
+ from pathlib import Path
2
+ from types import SimpleNamespace
3
+ from typing import Literal, TypeAlias, get_args
4
+
5
+ import click
6
+ from synth_ai.task_app_cfgs import LocalTaskAppConfig, ModalTaskAppConfig
7
+ from synth_ai.utils.cli import PromptedChoiceOption, PromptedChoiceType, PromptedPathOption
8
+ from synth_ai.utils.modal import deploy_modal_app, get_default_modal_bin_path
9
+ from synth_ai.utils.uvicorn import deploy_uvicorn_app
10
+
11
+ RuntimeType: TypeAlias = Literal[
12
+ "local",
13
+ "modal"
14
+ ]
15
+ RUNTIMES = get_args(RuntimeType)
16
+
17
+ MODAL_RUNTIME_OPTIONS = [
18
+ "task_app_name",
19
+ "cmd_arg",
20
+ "modal_bin_path",
21
+ "dry_run",
22
+ "modal_app_path",
23
+ ]
24
+ LOCAL_RUNTIME_OPTIONS = [
25
+ "trace",
26
+ "host",
27
+ "port"
28
+ ]
29
+
30
+ RUNTIME_MSG = SimpleNamespace(
31
+ init="[deploy]",
32
+ local="[deploy --runtime local]",
33
+ modal="[deploy --runtime modal]",
34
+ )
35
+
36
+
37
+ @click.command("deploy")
38
+ # --- Required options ---
39
+ @click.option(
40
+ "--task-app",
41
+ "task_app_path",
42
+ cls=PromptedPathOption,
43
+ type=click.Path(
44
+ exists=True,
45
+ dir_okay=False,
46
+ file_okay=True,
47
+ path_type=Path
48
+ ),
49
+ file_type=".py",
50
+ help=f"{RUNTIME_MSG.init} Enter the path to your task app",
51
+ )
52
+ @click.option(
53
+ "--runtime",
54
+ "runtime",
55
+ cls=PromptedChoiceOption,
56
+ type=PromptedChoiceType(RUNTIMES),
57
+ required=True
58
+ )
59
+ # --- Local-only options ---
60
+ @click.option(
61
+ "--trace/--no-trace",
62
+ "trace",
63
+ default=True,
64
+ help=f"{RUNTIME_MSG.local} Enable or disable trace output"
65
+ )
66
+ @click.option(
67
+ "--host",
68
+ "host",
69
+ default="127.0.0.1",
70
+ help=f"{RUNTIME_MSG.local} Host to bind to"
71
+ )
72
+ @click.option(
73
+ "--port",
74
+ "port",
75
+ default=8000,
76
+ type=int,
77
+ help=f"{RUNTIME_MSG.local} Port to bind to"
78
+ )
79
+ # --- Modal-only options ---
80
+ @click.option(
81
+ "--modal-app",
82
+ "modal_app_path",
83
+ cls=PromptedPathOption,
84
+ type=click.Path(
85
+ exists=True,
86
+ dir_okay=False,
87
+ file_okay=True,
88
+ path_type=Path
89
+ ),
90
+ file_type=".py",
91
+ prompt_guard=lambda ctx: (ctx.params.get("runtime") != "local"),
92
+ help=f"{RUNTIME_MSG.modal} Enter the path to your Modal app",
93
+ )
94
+ @click.option(
95
+ "--name",
96
+ "task_app_name",
97
+ default=None,
98
+ help=f"{RUNTIME_MSG.modal} Override Modal app name"
99
+ )
100
+ @click.option(
101
+ "--modal-mode",
102
+ "cmd_arg",
103
+ default="deploy",
104
+ help=f"{RUNTIME_MSG.modal} Mode: deploy or serve"
105
+ )
106
+ @click.option(
107
+ "--modal-cli",
108
+ "modal_bin_path",
109
+ type=click.Path(
110
+ dir_okay=False,
111
+ file_okay=True,
112
+ exists=True,
113
+ path_type=Path
114
+ ),
115
+ default=None,
116
+ help=f"{RUNTIME_MSG.modal} Path to Modal CLI",
117
+ )
118
+ @click.option(
119
+ "--dry-run",
120
+ "dry_run",
121
+ is_flag=True,
122
+ help=f"{RUNTIME_MSG.modal} Print Modal command without executing"
123
+ )
124
+ @click.option(
125
+ "--env-file",
126
+ "env_file",
127
+ multiple=True,
128
+ type=click.Path(exists=True),
129
+ help="Path to .env file(s) to load"
130
+ )
131
+ def deploy_cmd(
132
+ task_app_path: Path,
133
+ runtime: RuntimeType,
134
+ env_file: tuple[str, ...],
135
+ **kwargs
136
+ ) -> None:
137
+ """Deploy a task app to local or Modal runtime."""
138
+ match runtime:
139
+ case "local":
140
+ opts = {k: v for k, v in kwargs.items() if k in LOCAL_RUNTIME_OPTIONS}
141
+ deploy_uvicorn_app(LocalTaskAppConfig(**opts, task_app_path=task_app_path))
142
+
143
+ case "modal":
144
+ opts = {k: v for k, v in kwargs.items() if k in MODAL_RUNTIME_OPTIONS}
145
+
146
+ if "modal_app_path" not in opts or opts["modal_app_path"] is None:
147
+ raise click.ClickException("Modal app path required")
148
+
149
+ if opts["cmd_arg"] == "serve" and opts["dry_run"] is True:
150
+ raise click.ClickException("--modal-mode=serve cannot be combined with --dry-run")
151
+
152
+ modal_bin_path = opts.get("modal_bin_path") or get_default_modal_bin_path()
153
+ if not modal_bin_path:
154
+ raise click.ClickException(
155
+ "Modal CLI not found. Install the `modal` package or pass --modal-cli with its path."
156
+ )
157
+ if isinstance(modal_bin_path, str):
158
+ modal_bin_path = Path(modal_bin_path)
159
+ opts["modal_bin_path"] = modal_bin_path
160
+ deploy_modal_app(ModalTaskAppConfig(**opts, task_app_path=task_app_path))
161
+
162
+ __all__ = ["deploy_cmd"]
@@ -253,7 +253,7 @@ def view(url: str):
253
253
 
254
254
  @cli.command()
255
255
  @click.option("--db-file", default="synth_ai.db", help="Database file path")
256
- @click.option("--sqld-port", default=8080, type=int, help="Port for sqld HTTP interface")
256
+ @click.option("--sqld-port", default=8080, type=int, help="Port for sqld Hrana WebSocket interface (HTTP API will be port+1)")
257
257
  @click.option("--env-port", default=8901, type=int, help="Port for environment service")
258
258
  @click.option("--no-sqld", is_flag=True, help="Skip starting sqld daemon")
259
259
  @click.option("--no-env", is_flag=True, help="Skip starting environment service")
@@ -298,32 +298,37 @@ def serve(
298
298
 
299
299
  # Start sqld if requested
300
300
  if not no_sqld:
301
+ hrana_port = sqld_port
302
+ http_port = sqld_port + 1
301
303
  # Check if sqld is already running
302
304
  try:
303
305
  result = subprocess.run(
304
- ["pgrep", "-f", f"sqld.*--http-listen-addr.*:{sqld_port}"],
306
+ ["pgrep", "-f", f"sqld.*(--hrana-listen-addr.*:{hrana_port}|--http-listen-addr.*:{http_port})"],
305
307
  capture_output=True,
306
308
  text=True,
307
309
  )
308
310
  if result.returncode == 0:
309
- click.echo(f"✅ sqld already running on port {sqld_port}")
311
+ click.echo(f"✅ sqld already running on hrana port {hrana_port}, HTTP API port {http_port}")
310
312
  click.echo(f" Database: {db_file}")
311
- click.echo(f" HTTP API: http://127.0.0.1:{sqld_port}")
313
+ click.echo(f" libsql: libsql://127.0.0.1:{hrana_port}")
314
+ click.echo(f" HTTP API: http://127.0.0.1:{http_port}")
312
315
  else:
313
316
  # Find or install sqld
314
317
  sqld_bin = find_sqld_binary()
315
318
  if not sqld_bin:
316
319
  sqld_bin = install_sqld()
317
320
 
318
- click.echo(f"🗄️ Starting sqld (local only) on port {sqld_port}")
321
+ click.echo(f"🗄️ Starting sqld (local only) on hrana port {hrana_port}, HTTP API port {http_port}")
319
322
 
320
323
  # Start sqld
321
324
  sqld_cmd = [
322
325
  sqld_bin,
323
326
  "--db-path",
324
327
  db_file,
328
+ "--hrana-listen-addr",
329
+ f"127.0.0.1:{hrana_port}",
325
330
  "--http-listen-addr",
326
- f"127.0.0.1:{sqld_port}",
331
+ f"127.0.0.1:{http_port}",
327
332
  ]
328
333
 
329
334
  # Create log file
@@ -346,7 +351,8 @@ def serve(
346
351
 
347
352
  click.echo("✅ sqld started successfully!")
348
353
  click.echo(f" Database: {db_file}")
349
- click.echo(f" HTTP API: http://127.0.0.1:{sqld_port}")
354
+ click.echo(f" libsql: libsql://127.0.0.1:{hrana_port}")
355
+ click.echo(f" HTTP API: http://127.0.0.1:{http_port}")
350
356
  click.echo(f" Log file: {os.path.abspath('sqld.log')}")
351
357
 
352
358
  except FileNotFoundError:
@@ -417,7 +423,7 @@ def serve(
417
423
  click.echo(f" Working directory: {os.getcwd()}")
418
424
  click.echo("")
419
425
  click.echo("🔄 Starting services...")
420
- click.echo(f" - sqld daemon: http://127.0.0.1:{sqld_port}")
426
+ click.echo(f" - sqld daemon: libsql://127.0.0.1:{sqld_port} (HTTP API: http://127.0.0.1:{sqld_port + 1})")
421
427
  click.echo(f" - Environment service: http://127.0.0.1:{env_port}")
422
428
  click.echo("")
423
429
  click.echo("💡 Tips:")