synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.9.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synth-ai might be problematic. Click here for more details.
- examples/common_old/backend.py +0 -1
- examples/crafter_debug_render.py +15 -6
- examples/evals_old/compare_models.py +1 -0
- examples/finetuning_old/_backup_synth_qwen/filter_traces_achievements.py +6 -2
- examples/finetuning_old/_backup_synth_qwen/react_agent_lm.py +4 -4
- examples/finetuning_old/_backup_synth_qwen/sft_kickoff.py +4 -3
- examples/finetuning_old/synth_qwen_v1/filter_traces_achievements.py +6 -2
- examples/finetuning_old/synth_qwen_v1/finetune.py +1 -1
- examples/finetuning_old/synth_qwen_v1/hello_ft_model.py +4 -4
- examples/finetuning_old/synth_qwen_v1/infer.py +1 -2
- examples/finetuning_old/synth_qwen_v1/poll.py +4 -2
- examples/finetuning_old/synth_qwen_v1/prepare_data.py +8 -8
- examples/finetuning_old/synth_qwen_v1/react_agent_lm.py +5 -4
- examples/finetuning_old/synth_qwen_v1/run_crafter_sft_job.py +11 -8
- examples/finetuning_old/synth_qwen_v1/run_ft_job.py +17 -12
- examples/finetuning_old/synth_qwen_v1/upload_data.py +1 -1
- examples/finetuning_old/synth_qwen_v1/util.py +7 -2
- examples/rl/configs/eval_base_qwen.toml +1 -1
- examples/rl/configs/rl_from_base_qwen17.toml +1 -1
- examples/rl/download_dataset.py +26 -10
- examples/rl/run_eval.py +17 -15
- examples/rl/run_rl_and_save.py +24 -7
- examples/rl/task_app/math_single_step.py +128 -11
- examples/rl/task_app/math_task_app.py +11 -3
- examples/rl_old/task_app.py +222 -53
- examples/warming_up_to_rl/analyze_trace_db.py +7 -5
- examples/warming_up_to_rl/export_trace_sft.py +141 -16
- examples/warming_up_to_rl/groq_test.py +11 -4
- examples/warming_up_to_rl/manage_secrets.py +15 -6
- examples/warming_up_to_rl/readme.md +9 -2
- examples/warming_up_to_rl/run_eval.py +108 -30
- examples/warming_up_to_rl/run_fft_and_save.py +128 -52
- examples/warming_up_to_rl/run_local_rollout.py +87 -36
- examples/warming_up_to_rl/run_local_rollout_modal.py +113 -25
- examples/warming_up_to_rl/run_local_rollout_parallel.py +80 -16
- examples/warming_up_to_rl/run_local_rollout_traced.py +125 -20
- examples/warming_up_to_rl/run_rl_and_save.py +31 -7
- examples/warming_up_to_rl/run_rollout_remote.py +37 -10
- examples/warming_up_to_rl/task_app/grpo_crafter.py +90 -27
- examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py +9 -27
- examples/warming_up_to_rl/task_app/synth_envs_hosted/environment_routes.py +46 -108
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/app.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py +50 -17
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py +35 -21
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/react_agent.py +8 -4
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/shared.py +29 -26
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/tools.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py +17 -13
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py +106 -63
- examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py +82 -84
- examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py +76 -59
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/volume.py +43 -49
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_service.py +5 -15
- synth_ai/__init__.py +1 -0
- synth_ai/api/train/builders.py +34 -10
- synth_ai/api/train/cli.py +172 -32
- synth_ai/api/train/config_finder.py +59 -4
- synth_ai/api/train/env_resolver.py +32 -14
- synth_ai/api/train/pollers.py +11 -3
- synth_ai/api/train/task_app.py +4 -1
- synth_ai/api/train/utils.py +20 -4
- synth_ai/cli/__init__.py +11 -4
- synth_ai/cli/balance.py +1 -1
- synth_ai/cli/demo.py +19 -5
- synth_ai/cli/rl_demo.py +75 -16
- synth_ai/cli/root.py +116 -37
- synth_ai/cli/task_apps.py +1276 -186
- synth_ai/cli/traces.py +1 -0
- synth_ai/cli/turso.py +73 -0
- synth_ai/core/experiment.py +0 -2
- synth_ai/demo_registry.py +67 -30
- synth_ai/demos/core/cli.py +493 -164
- synth_ai/demos/demo_task_apps/core.py +50 -6
- synth_ai/demos/demo_task_apps/crafter/configs/crafter_fft_4b.toml +2 -3
- synth_ai/demos/demo_task_apps/crafter/grpo_crafter_task_app.py +36 -28
- synth_ai/demos/demo_task_apps/math/_common.py +1 -2
- synth_ai/demos/demo_task_apps/math/deploy_modal.py +0 -2
- synth_ai/demos/demo_task_apps/math/modal_task_app.py +168 -65
- synth_ai/demos/demo_task_apps/math/task_app_entry.py +0 -1
- synth_ai/environments/examples/bandit/engine.py +12 -4
- synth_ai/environments/examples/bandit/taskset.py +4 -4
- synth_ai/environments/reproducibility/tree.py +3 -1
- synth_ai/environments/service/core_routes.py +6 -2
- synth_ai/evals/base.py +0 -2
- synth_ai/experimental/synth_oss.py +11 -12
- synth_ai/handshake.py +3 -1
- synth_ai/http_client.py +31 -7
- synth_ai/inference/__init__.py +0 -2
- synth_ai/inference/client.py +8 -4
- synth_ai/jobs/client.py +40 -10
- synth_ai/learning/client.py +33 -8
- synth_ai/learning/config.py +0 -2
- synth_ai/learning/constants.py +0 -2
- synth_ai/learning/ft_client.py +6 -3
- synth_ai/learning/health.py +9 -2
- synth_ai/learning/jobs.py +17 -5
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +1 -3
- synth_ai/learning/prompts/random_search.py +4 -1
- synth_ai/learning/prompts/run_random_search_banking77.py +6 -1
- synth_ai/learning/rl_client.py +42 -14
- synth_ai/learning/sse.py +0 -2
- synth_ai/learning/validators.py +6 -2
- synth_ai/lm/caching/ephemeral.py +1 -3
- synth_ai/lm/core/exceptions.py +0 -2
- synth_ai/lm/core/main.py +13 -1
- synth_ai/lm/core/synth_models.py +0 -1
- synth_ai/lm/core/vendor_clients.py +4 -2
- synth_ai/lm/overrides.py +2 -2
- synth_ai/lm/vendors/core/anthropic_api.py +7 -7
- synth_ai/lm/vendors/core/openai_api.py +2 -0
- synth_ai/lm/vendors/openai_standard.py +3 -1
- synth_ai/lm/vendors/openai_standard_responses.py +6 -3
- synth_ai/lm/vendors/supported/custom_endpoint.py +1 -3
- synth_ai/lm/vendors/synth_client.py +37 -10
- synth_ai/rl/__init__.py +0 -1
- synth_ai/rl/contracts.py +0 -2
- synth_ai/rl/env_keys.py +6 -1
- synth_ai/task/__init__.py +1 -0
- synth_ai/task/apps/__init__.py +11 -11
- synth_ai/task/auth.py +29 -17
- synth_ai/task/client.py +3 -1
- synth_ai/task/contracts.py +1 -0
- synth_ai/task/datasets.py +3 -1
- synth_ai/task/errors.py +3 -2
- synth_ai/task/health.py +0 -2
- synth_ai/task/json.py +0 -1
- synth_ai/task/proxy.py +2 -5
- synth_ai/task/rubrics.py +9 -3
- synth_ai/task/server.py +31 -5
- synth_ai/task/tracing_utils.py +8 -3
- synth_ai/task/validators.py +0 -1
- synth_ai/task/vendors.py +0 -1
- synth_ai/tracing_v3/db_config.py +26 -1
- synth_ai/tracing_v3/decorators.py +1 -0
- synth_ai/tracing_v3/examples/basic_usage.py +3 -2
- synth_ai/tracing_v3/hooks.py +2 -0
- synth_ai/tracing_v3/replica_sync.py +1 -0
- synth_ai/tracing_v3/session_tracer.py +24 -3
- synth_ai/tracing_v3/storage/base.py +4 -1
- synth_ai/tracing_v3/storage/factory.py +0 -1
- synth_ai/tracing_v3/turso/manager.py +102 -38
- synth_ai/tracing_v3/turso/models.py +4 -1
- synth_ai/tracing_v3/utils.py +1 -0
- synth_ai/v0/tracing/upload.py +32 -135
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/METADATA +1 -1
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/RECORD +154 -154
- synth_ai/install_sqld.sh +0 -40
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/top_level.txt +0 -0
synth_ai/learning/rl_client.py
CHANGED
|
@@ -31,10 +31,17 @@ class RlClient:
|
|
|
31
31
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=30.0) as http:
|
|
32
32
|
js = await http.get(path)
|
|
33
33
|
if not isinstance(js, dict):
|
|
34
|
-
raise HTTPError(
|
|
34
|
+
raise HTTPError(
|
|
35
|
+
status=500, url=path, message="invalid_service_response", body_snippet=str(js)[:200]
|
|
36
|
+
)
|
|
35
37
|
start_url = js.get("training_start_url")
|
|
36
38
|
if not isinstance(start_url, str) or not start_url:
|
|
37
|
-
raise HTTPError(
|
|
39
|
+
raise HTTPError(
|
|
40
|
+
status=500,
|
|
41
|
+
url=path,
|
|
42
|
+
message="missing_training_start_url",
|
|
43
|
+
body_snippet=str(js)[:200],
|
|
44
|
+
)
|
|
38
45
|
return start_url
|
|
39
46
|
|
|
40
47
|
async def create_job(
|
|
@@ -63,7 +70,12 @@ class RlClient:
|
|
|
63
70
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
|
|
64
71
|
js = await http.post_json(f"{_api_base(self._base_url)}/rl/jobs", json=body)
|
|
65
72
|
if not isinstance(js, dict):
|
|
66
|
-
raise HTTPError(
|
|
73
|
+
raise HTTPError(
|
|
74
|
+
status=500,
|
|
75
|
+
url="/api/rl/jobs",
|
|
76
|
+
message="invalid_create_response",
|
|
77
|
+
body_snippet=str(js)[:200],
|
|
78
|
+
)
|
|
67
79
|
return js
|
|
68
80
|
|
|
69
81
|
async def start_job_if_supported(self, job_id: str) -> Optional[Dict[str, Any]]:
|
|
@@ -80,11 +92,15 @@ class RlClient:
|
|
|
80
92
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=30.0) as http:
|
|
81
93
|
return await http.get(f"{_api_base(self._base_url)}/learning/jobs/{job_id}")
|
|
82
94
|
|
|
83
|
-
async def get_events(
|
|
95
|
+
async def get_events(
|
|
96
|
+
self, job_id: str, *, since_seq: int = 0, limit: int = 200
|
|
97
|
+
) -> List[Dict[str, Any]]:
|
|
84
98
|
params = {"since_seq": since_seq, "limit": limit}
|
|
85
99
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=30.0) as http:
|
|
86
100
|
try:
|
|
87
|
-
js = await http.get(
|
|
101
|
+
js = await http.get(
|
|
102
|
+
f"{_api_base(self._base_url)}/learning/jobs/{job_id}/events", params=params
|
|
103
|
+
)
|
|
88
104
|
except HTTPError as he:
|
|
89
105
|
try:
|
|
90
106
|
print(
|
|
@@ -99,10 +115,14 @@ class RlClient:
|
|
|
99
115
|
return evs
|
|
100
116
|
return []
|
|
101
117
|
|
|
102
|
-
async def get_metrics(
|
|
118
|
+
async def get_metrics(
|
|
119
|
+
self, job_id: str, *, after_step: int = -1, limit: int = 200
|
|
120
|
+
) -> List[Dict[str, Any]]:
|
|
103
121
|
params = {"after_step": after_step, "limit": limit}
|
|
104
122
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=30.0) as http:
|
|
105
|
-
js = await http.get(
|
|
123
|
+
js = await http.get(
|
|
124
|
+
f"{_api_base(self._base_url)}/learning/jobs/{job_id}/metrics", params=params
|
|
125
|
+
)
|
|
106
126
|
if isinstance(js, dict) and isinstance(js.get("points"), list):
|
|
107
127
|
return js["points"]
|
|
108
128
|
return []
|
|
@@ -161,7 +181,9 @@ class RlClient:
|
|
|
161
181
|
if events_job_id and events_job_id not in stream_ids:
|
|
162
182
|
stream_ids.append(events_job_id)
|
|
163
183
|
try:
|
|
164
|
-
print(
|
|
184
|
+
print(
|
|
185
|
+
f"[poll] streams={stream_ids} intervals={interval_seconds}s since_map={last_seq_by_stream} empty_polls={empty_polls}"
|
|
186
|
+
)
|
|
165
187
|
except Exception:
|
|
166
188
|
pass
|
|
167
189
|
total_events_this_cycle = 0
|
|
@@ -173,13 +195,17 @@ class RlClient:
|
|
|
173
195
|
events = await self.get_events(ev_id, since_seq=since, limit=200)
|
|
174
196
|
except HTTPError as he:
|
|
175
197
|
try:
|
|
176
|
-
print(
|
|
198
|
+
print(
|
|
199
|
+
f"[poll] get_events error status={he.status} url={he.url} since={since} body={(he.body_snippet or '')[:200]}"
|
|
200
|
+
)
|
|
177
201
|
except Exception:
|
|
178
202
|
pass
|
|
179
203
|
events = []
|
|
180
204
|
except Exception as e:
|
|
181
205
|
try:
|
|
182
|
-
print(
|
|
206
|
+
print(
|
|
207
|
+
f"[poll] get_events unexpected error ev_id={ev_id} since={since} err={type(e).__name__}: {e}"
|
|
208
|
+
)
|
|
183
209
|
except Exception:
|
|
184
210
|
pass
|
|
185
211
|
events = []
|
|
@@ -238,7 +264,9 @@ class RlClient:
|
|
|
238
264
|
)
|
|
239
265
|
except Exception:
|
|
240
266
|
pass
|
|
241
|
-
raise AssertionError(
|
|
267
|
+
raise AssertionError(
|
|
268
|
+
f"No new events detected for {empty_polls_threshold} consecutive polls. Check event ingestion."
|
|
269
|
+
)
|
|
242
270
|
|
|
243
271
|
if not saw_any_event and (time.time() - start_t) > int(startup_deadline_s):
|
|
244
272
|
try:
|
|
@@ -247,10 +275,10 @@ class RlClient:
|
|
|
247
275
|
)
|
|
248
276
|
except Exception:
|
|
249
277
|
pass
|
|
250
|
-
raise AssertionError(
|
|
278
|
+
raise AssertionError(
|
|
279
|
+
f"No events observed within startup window ({startup_deadline_s}s). Investigate event streaming."
|
|
280
|
+
)
|
|
251
281
|
|
|
252
282
|
await sleep(interval_seconds)
|
|
253
283
|
if max_seconds is not None and (time.time() - start_t) >= max_seconds:
|
|
254
284
|
raise TimeoutError(f"Polling timed out after {max_seconds}s for job {job_id}")
|
|
255
|
-
|
|
256
|
-
|
synth_ai/learning/sse.py
CHANGED
synth_ai/learning/validators.py
CHANGED
|
@@ -13,7 +13,7 @@ def validate_training_jsonl(path: str | Path, *, sample_lines: int = 50) -> None
|
|
|
13
13
|
lines = p.read_text().splitlines()
|
|
14
14
|
if not lines:
|
|
15
15
|
raise ValueError("empty JSONL")
|
|
16
|
-
for i, line in enumerate(lines[: max(1, sample_lines)
|
|
16
|
+
for i, line in enumerate(lines[: max(1, sample_lines)], start=1):
|
|
17
17
|
if not line.strip():
|
|
18
18
|
continue
|
|
19
19
|
try:
|
|
@@ -29,7 +29,11 @@ def validate_training_jsonl(path: str | Path, *, sample_lines: int = 50) -> None
|
|
|
29
29
|
for m in msgs:
|
|
30
30
|
if not isinstance(m, dict):
|
|
31
31
|
raise ValueError(f"line {i}: non-dict message")
|
|
32
|
-
if
|
|
32
|
+
if (
|
|
33
|
+
not isinstance(m.get("role"), str)
|
|
34
|
+
or not isinstance(m.get("content"), str)
|
|
35
|
+
or not m["content"].strip()
|
|
36
|
+
):
|
|
33
37
|
raise ValueError(f"line {i}: invalid role/content")
|
|
34
38
|
|
|
35
39
|
|
synth_ai/lm/caching/ephemeral.py
CHANGED
|
@@ -28,9 +28,7 @@ class EphemeralCache:
|
|
|
28
28
|
os.makedirs(fast_cache_dir, exist_ok=True)
|
|
29
29
|
self.fast_cache = Cache(fast_cache_dir, size_limit=DISKCACHE_SIZE_LIMIT)
|
|
30
30
|
|
|
31
|
-
def hit_cache(
|
|
32
|
-
self, key: str, response_model: BaseModel | None = None
|
|
33
|
-
) -> BaseLMResponse | None:
|
|
31
|
+
def hit_cache(self, key: str, response_model: BaseModel | None = None) -> BaseLMResponse | None:
|
|
34
32
|
"""
|
|
35
33
|
Check if a response exists in cache for the given key.
|
|
36
34
|
|
synth_ai/lm/core/exceptions.py
CHANGED
synth_ai/lm/core/main.py
CHANGED
|
@@ -113,7 +113,19 @@ class LM:
|
|
|
113
113
|
max_retries: Literal["None", "Few", "Many"] = "Few",
|
|
114
114
|
structured_output_mode: Literal["stringified_json", "forced_json"] = "stringified_json",
|
|
115
115
|
synth_logging: bool = True,
|
|
116
|
-
provider: Literal[
|
|
116
|
+
provider: Literal[
|
|
117
|
+
"openai",
|
|
118
|
+
"anthropic",
|
|
119
|
+
"groq",
|
|
120
|
+
"gemini",
|
|
121
|
+
"deepseek",
|
|
122
|
+
"grok",
|
|
123
|
+
"mistral",
|
|
124
|
+
"openrouter",
|
|
125
|
+
"together",
|
|
126
|
+
]
|
|
127
|
+
| str
|
|
128
|
+
| None = None,
|
|
117
129
|
enable_thinking: bool | None = None,
|
|
118
130
|
):
|
|
119
131
|
# print("Structured output mode", structured_output_mode)
|
synth_ai/lm/core/synth_models.py
CHANGED
|
@@ -180,8 +180,10 @@ def get_client(
|
|
|
180
180
|
elif any(regex.match(model_name) for regex in custom_endpoint_naming_regexes):
|
|
181
181
|
# Custom endpoints are passed as the endpoint URL
|
|
182
182
|
return CustomEndpointClient(endpoint_url=model_name)
|
|
183
|
-
elif (
|
|
184
|
-
|
|
183
|
+
elif (
|
|
184
|
+
any(regex.match(model_name) for regex in synth_naming_regexes)
|
|
185
|
+
or model_name in SYNTH_SUPPORTED_MODELS
|
|
186
|
+
):
|
|
185
187
|
# Synth models use OpenAI-compatible client with custom endpoint
|
|
186
188
|
return OpenAIStructuredOutputClient(synth_logging=synth_logging)
|
|
187
189
|
else:
|
synth_ai/lm/overrides.py
CHANGED
|
@@ -20,8 +20,8 @@ from synth_ai.lm.injection import (
|
|
|
20
20
|
# "params": { ... api params to override ... },
|
|
21
21
|
# "tools": { ... optional tools overrides ... },
|
|
22
22
|
# }
|
|
23
|
-
_override_specs_ctx: contextvars.ContextVar[list[dict[str, Any]] | None] = (
|
|
24
|
-
|
|
23
|
+
_override_specs_ctx: contextvars.ContextVar[list[dict[str, Any]] | None] = contextvars.ContextVar(
|
|
24
|
+
"override_specs", default=None
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
# ContextVars actually applied for the specific call once matched
|
|
@@ -100,13 +100,13 @@ class AnthropicAPI(VendorBase):
|
|
|
100
100
|
and model in CLAUDE_REASONING_MODELS
|
|
101
101
|
and reasoning_effort in ["high", "medium"]
|
|
102
102
|
):
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
budget = SONNET_37_BUDGETS[reasoning_effort]
|
|
104
|
+
api_params["thinking"] = {
|
|
105
|
+
"type": "enabled",
|
|
106
|
+
"budget_tokens": budget,
|
|
107
|
+
}
|
|
108
|
+
api_params["max_tokens"] = budget + 4096
|
|
109
|
+
api_params["temperature"] = 1
|
|
110
110
|
except (ImportError, AttributeError, TypeError):
|
|
111
111
|
pass
|
|
112
112
|
|
|
@@ -74,11 +74,13 @@ class OpenAIStructuredOutputClient(OpenAIStandard):
|
|
|
74
74
|
elif synth_logging:
|
|
75
75
|
# print("Using synth logging - OpenAIStructuredOutputClient")
|
|
76
76
|
from synth_ai.lm.provider_support.openai import AsyncOpenAI, OpenAI
|
|
77
|
+
|
|
77
78
|
sync_client = OpenAI()
|
|
78
79
|
async_client = AsyncOpenAI()
|
|
79
80
|
else:
|
|
80
81
|
# print("Not using synth logging - OpenAIStructuredOutputClient")
|
|
81
82
|
from openai import AsyncOpenAI, OpenAI
|
|
83
|
+
|
|
82
84
|
sync_client = OpenAI()
|
|
83
85
|
async_client = AsyncOpenAI()
|
|
84
86
|
|
|
@@ -427,7 +427,9 @@ class OpenAIStandard(VendorBase, OpenAIResponsesAPIMixin):
|
|
|
427
427
|
"type": getattr(tc, "type", "function"),
|
|
428
428
|
"function": {
|
|
429
429
|
"name": getattr(getattr(tc, "function", None), "name", None),
|
|
430
|
-
"arguments": getattr(
|
|
430
|
+
"arguments": getattr(
|
|
431
|
+
getattr(tc, "function", None), "arguments", None
|
|
432
|
+
),
|
|
431
433
|
},
|
|
432
434
|
}
|
|
433
435
|
)
|
|
@@ -223,7 +223,9 @@ class OpenAIResponsesAPIMixin:
|
|
|
223
223
|
if not synth_gpu_endpoint:
|
|
224
224
|
raise ValueError("SYNTH_GPU_HARMONY_ENDPOINT environment variable not set")
|
|
225
225
|
|
|
226
|
-
async with
|
|
226
|
+
async with (
|
|
227
|
+
aiohttp.ClientSession() as session,
|
|
228
|
+
session.post(
|
|
227
229
|
f"{synth_gpu_endpoint}/v1/completions",
|
|
228
230
|
json={
|
|
229
231
|
"model": model,
|
|
@@ -231,8 +233,9 @@ class OpenAIResponsesAPIMixin:
|
|
|
231
233
|
"max_tokens": lm_config.get("max_tokens", 4096),
|
|
232
234
|
"temperature": lm_config.get("temperature", 0.8),
|
|
233
235
|
},
|
|
234
|
-
) as resp
|
|
235
|
-
|
|
236
|
+
) as resp,
|
|
237
|
+
):
|
|
238
|
+
result = await resp.json()
|
|
236
239
|
|
|
237
240
|
# Parse response using Harmony
|
|
238
241
|
response_tokens = result.get("choices", [{}])[0].get("text", "")
|
|
@@ -194,9 +194,7 @@ IMPORTANT: To use a tool, respond with JSON wrapped in ```json fences:
|
|
|
194
194
|
|
|
195
195
|
For regular responses, just respond normally without JSON fences."""
|
|
196
196
|
|
|
197
|
-
def _extract_tool_calls(
|
|
198
|
-
self, content: str, tools: list[BaseTool]
|
|
199
|
-
) -> tuple[list | None, str]:
|
|
197
|
+
def _extract_tool_calls(self, content: str, tools: list[BaseTool]) -> tuple[list | None, str]:
|
|
200
198
|
"""Extract and validate tool calls from response."""
|
|
201
199
|
# Look for JSON fenced blocks
|
|
202
200
|
json_pattern = r"```json\s*(\{.*?\})\s*```"
|
|
@@ -276,8 +276,20 @@ class AsyncSynthClient:
|
|
|
276
276
|
This method provides the OpenAI client interface structure.
|
|
277
277
|
"""
|
|
278
278
|
return await self._chat_completions_create(
|
|
279
|
-
model,
|
|
280
|
-
|
|
279
|
+
model,
|
|
280
|
+
messages,
|
|
281
|
+
temperature,
|
|
282
|
+
max_tokens,
|
|
283
|
+
top_p,
|
|
284
|
+
frequency_penalty,
|
|
285
|
+
presence_penalty,
|
|
286
|
+
stop,
|
|
287
|
+
stream,
|
|
288
|
+
tools,
|
|
289
|
+
tool_choice,
|
|
290
|
+
response_format,
|
|
291
|
+
seed,
|
|
292
|
+
**kwargs,
|
|
281
293
|
)
|
|
282
294
|
|
|
283
295
|
async def _chat_completions_create(
|
|
@@ -368,7 +380,8 @@ class AsyncSynthClient:
|
|
|
368
380
|
if isinstance(bt, int) and isinstance(mt, int) and bt > mt:
|
|
369
381
|
logger.warning(
|
|
370
382
|
"thinking_budget (%s) exceeds max_tokens (%s) – forwarding as-is",
|
|
371
|
-
str(bt),
|
|
383
|
+
str(bt),
|
|
384
|
+
str(mt),
|
|
372
385
|
)
|
|
373
386
|
except Exception:
|
|
374
387
|
pass
|
|
@@ -387,6 +400,7 @@ class AsyncSynthClient:
|
|
|
387
400
|
|
|
388
401
|
# If streaming requested, return an async stream adapter
|
|
389
402
|
if stream:
|
|
403
|
+
|
|
390
404
|
async def _astream():
|
|
391
405
|
await self._ensure_client()
|
|
392
406
|
async with self._client.stream("POST", url, json=payload) as r: # type: ignore
|
|
@@ -678,6 +692,7 @@ def create_sync_client(config: SynthConfig | None = None) -> SyncSynthClient:
|
|
|
678
692
|
# Drop-in replacements for OpenAI clients
|
|
679
693
|
# These allow Synth to be used as a complete replacement for OpenAI
|
|
680
694
|
|
|
695
|
+
|
|
681
696
|
class AsyncOpenAI(AsyncSynthClient):
|
|
682
697
|
"""
|
|
683
698
|
Drop-in replacement for openai.AsyncOpenAI.
|
|
@@ -710,16 +725,22 @@ class AsyncOpenAI(AsyncSynthClient):
|
|
|
710
725
|
"""
|
|
711
726
|
# Handle OpenAI-style initialization
|
|
712
727
|
from ..config import SynthConfig
|
|
728
|
+
|
|
713
729
|
if api_key or base_url:
|
|
714
730
|
config = SynthConfig(
|
|
715
|
-
base_url=base_url
|
|
716
|
-
|
|
731
|
+
base_url=base_url
|
|
732
|
+
or os.getenv(
|
|
733
|
+
"OPENAI_API_BASE", "https://synth-backend-dev-docker.onrender.com/api"
|
|
734
|
+
),
|
|
735
|
+
api_key=api_key or os.getenv("OPENAI_API_KEY", ""),
|
|
717
736
|
)
|
|
718
737
|
else:
|
|
719
738
|
# Fallback to environment variables (OPENAI_* first, then SYNTH_*)
|
|
720
739
|
env_base = os.getenv("OPENAI_API_BASE") or os.getenv("SYNTH_BASE_URL")
|
|
721
740
|
env_key = os.getenv("OPENAI_API_KEY") or os.getenv("SYNTH_API_KEY")
|
|
722
|
-
config =
|
|
741
|
+
config = (
|
|
742
|
+
SynthConfig(base_url=env_base, api_key=env_key) if env_base and env_key else None
|
|
743
|
+
)
|
|
723
744
|
|
|
724
745
|
super().__init__(config, **kwargs)
|
|
725
746
|
|
|
@@ -742,15 +763,21 @@ class OpenAI(SyncSynthClient):
|
|
|
742
763
|
"""
|
|
743
764
|
# Handle OpenAI-style initialization
|
|
744
765
|
from ..config import SynthConfig
|
|
766
|
+
|
|
745
767
|
if api_key or base_url:
|
|
746
768
|
config = SynthConfig(
|
|
747
|
-
base_url=base_url
|
|
748
|
-
|
|
769
|
+
base_url=base_url
|
|
770
|
+
or os.getenv(
|
|
771
|
+
"OPENAI_API_BASE", "https://synth-backend-dev-docker.onrender.com/api"
|
|
772
|
+
),
|
|
773
|
+
api_key=api_key or os.getenv("OPENAI_API_KEY", ""),
|
|
749
774
|
)
|
|
750
775
|
else:
|
|
751
776
|
env_base = os.getenv("OPENAI_API_BASE") or os.getenv("SYNTH_BASE_URL")
|
|
752
777
|
env_key = os.getenv("OPENAI_API_KEY") or os.getenv("SYNTH_API_KEY")
|
|
753
|
-
config =
|
|
778
|
+
config = (
|
|
779
|
+
SynthConfig(base_url=env_base, api_key=env_key) if env_base and env_key else None
|
|
780
|
+
)
|
|
754
781
|
|
|
755
782
|
super().__init__(config, **kwargs)
|
|
756
783
|
|
|
@@ -760,7 +787,7 @@ __all__ = [
|
|
|
760
787
|
"AsyncSynthClient",
|
|
761
788
|
"SyncSynthClient",
|
|
762
789
|
"AsyncOpenAI", # Drop-in replacement for openai.AsyncOpenAI
|
|
763
|
-
"OpenAI",
|
|
790
|
+
"OpenAI", # Drop-in replacement for openai.OpenAI
|
|
764
791
|
"create_async_client",
|
|
765
792
|
"create_sync_client",
|
|
766
793
|
"create_chat_completion_async",
|
synth_ai/rl/__init__.py
CHANGED
synth_ai/rl/contracts.py
CHANGED
synth_ai/rl/env_keys.py
CHANGED
|
@@ -101,7 +101,12 @@ def setup_environment_api_key(
|
|
|
101
101
|
|
|
102
102
|
body = {"name": "ENVIRONMENT_API_KEY", "ciphertext_b64": ciphertext_b64}
|
|
103
103
|
post_url = f"{backend}/api/v1/env-keys"
|
|
104
|
-
response2 = requests.post(
|
|
104
|
+
response2 = requests.post(
|
|
105
|
+
post_url,
|
|
106
|
+
headers={**headers, "Content-Type": "application/json"},
|
|
107
|
+
json=body,
|
|
108
|
+
timeout=timeout,
|
|
109
|
+
)
|
|
105
110
|
_raise_with_detail(response2)
|
|
106
111
|
|
|
107
112
|
try:
|
synth_ai/task/__init__.py
CHANGED
synth_ai/task/apps/__init__.py
CHANGED
|
@@ -68,7 +68,7 @@ class TaskAppRegistry:
|
|
|
68
68
|
|
|
69
69
|
def __iter__(self) -> Iterable[TaskAppEntry]:
|
|
70
70
|
return iter(self.list())
|
|
71
|
-
|
|
71
|
+
|
|
72
72
|
def clear(self) -> None:
|
|
73
73
|
"""Clear all registered task apps."""
|
|
74
74
|
self._entries.clear()
|
|
@@ -85,34 +85,34 @@ def register_task_app(*, entry: TaskAppEntry) -> None:
|
|
|
85
85
|
def discover_task_apps_from_cwd() -> None:
|
|
86
86
|
"""Discover and register task apps from the current working directory and subdirectories."""
|
|
87
87
|
cwd = Path.cwd()
|
|
88
|
-
|
|
88
|
+
|
|
89
89
|
# Look for task app files in common patterns
|
|
90
90
|
patterns = [
|
|
91
91
|
"**/task_app/*.py",
|
|
92
|
-
"**/task_apps/*.py",
|
|
92
|
+
"**/task_apps/*.py",
|
|
93
93
|
"**/*_task_app.py",
|
|
94
94
|
"**/grpo_crafter.py",
|
|
95
95
|
"**/math_single_step.py",
|
|
96
96
|
]
|
|
97
|
-
|
|
97
|
+
|
|
98
98
|
discovered_files = []
|
|
99
99
|
for pattern in patterns:
|
|
100
100
|
discovered_files.extend(cwd.glob(pattern))
|
|
101
|
-
|
|
101
|
+
|
|
102
102
|
# Add current directory to Python path temporarily
|
|
103
103
|
original_path = sys.path.copy()
|
|
104
104
|
try:
|
|
105
105
|
sys.path.insert(0, str(cwd))
|
|
106
|
-
|
|
106
|
+
|
|
107
107
|
for file_path in discovered_files:
|
|
108
|
-
if file_path.name.startswith(
|
|
108
|
+
if file_path.name.startswith("__"):
|
|
109
109
|
continue
|
|
110
|
-
|
|
110
|
+
|
|
111
111
|
# Convert file path to module name
|
|
112
112
|
relative_path = file_path.relative_to(cwd)
|
|
113
113
|
module_parts = list(relative_path.parts[:-1]) + [relative_path.stem]
|
|
114
|
-
module_name =
|
|
115
|
-
|
|
114
|
+
module_name = ".".join(module_parts)
|
|
115
|
+
|
|
116
116
|
try:
|
|
117
117
|
# Import the module to trigger registration
|
|
118
118
|
importlib.import_module(module_name)
|
|
@@ -120,7 +120,7 @@ def discover_task_apps_from_cwd() -> None:
|
|
|
120
120
|
# Silently skip modules that can't be imported
|
|
121
121
|
# This allows for graceful handling of missing dependencies
|
|
122
122
|
continue
|
|
123
|
-
|
|
123
|
+
|
|
124
124
|
finally:
|
|
125
125
|
sys.path[:] = original_path
|
|
126
126
|
|
synth_ai/task/auth.py
CHANGED
|
@@ -12,7 +12,9 @@ _DEV_API_KEY_ENVS = ("dev_environment_api_key", "DEV_ENVIRONMENT_API_KEY")
|
|
|
12
12
|
_API_KEY_HEADER = "x-api-key"
|
|
13
13
|
_API_KEYS_HEADER = "x-api-keys"
|
|
14
14
|
_AUTH_HEADER = "authorization"
|
|
15
|
-
_API_KEY_ALIASES_ENV =
|
|
15
|
+
_API_KEY_ALIASES_ENV = (
|
|
16
|
+
"ENVIRONMENT_API_KEY_ALIASES" # comma-separated list of additional valid keys
|
|
17
|
+
)
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
def _mask(value: str, *, prefix: int = 4) -> str:
|
|
@@ -120,7 +122,9 @@ def require_api_key_dependency(request: Any) -> None:
|
|
|
120
122
|
|
|
121
123
|
allowed = allowed_environment_api_keys()
|
|
122
124
|
if not allowed:
|
|
123
|
-
raise http_exception(
|
|
125
|
+
raise http_exception(
|
|
126
|
+
503, "missing_environment_api_key", "ENVIRONMENT_API_KEY is not configured"
|
|
127
|
+
)
|
|
124
128
|
# Build candidate list for verbose diagnostics
|
|
125
129
|
single = list(_header_values(request, _API_KEY_HEADER))
|
|
126
130
|
multi = list(_header_values(request, _API_KEYS_HEADER))
|
|
@@ -132,22 +136,30 @@ def require_api_key_dependency(request: Any) -> None:
|
|
|
132
136
|
candidates = _split_csv(single + multi + bearer)
|
|
133
137
|
if not any(candidate in allowed for candidate in candidates):
|
|
134
138
|
try:
|
|
135
|
-
print(
|
|
136
|
-
|
|
139
|
+
print(
|
|
140
|
+
{
|
|
141
|
+
"task_auth_failed": True,
|
|
142
|
+
"allowed_first15": [k[:15] for k in allowed],
|
|
143
|
+
"allowed_count": len(allowed),
|
|
144
|
+
"got_first15": [c[:15] for c in candidates],
|
|
145
|
+
"got_lens": [len(c) for c in candidates],
|
|
146
|
+
"have_x_api_key": bool(single),
|
|
147
|
+
"have_x_api_keys": bool(multi),
|
|
148
|
+
"have_authorization": bool(auths),
|
|
149
|
+
},
|
|
150
|
+
flush=True,
|
|
151
|
+
)
|
|
152
|
+
except Exception:
|
|
153
|
+
pass
|
|
154
|
+
# Use 400 to make failures unmistakable during preflight
|
|
155
|
+
raise http_exception(
|
|
156
|
+
400,
|
|
157
|
+
"unauthorised",
|
|
158
|
+
"API key missing or invalid",
|
|
159
|
+
extra={
|
|
137
160
|
"allowed_first15": [k[:15] for k in allowed],
|
|
138
161
|
"allowed_count": len(allowed),
|
|
139
162
|
"got_first15": [c[:15] for c in candidates],
|
|
140
163
|
"got_lens": [len(c) for c in candidates],
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
"have_authorization": bool(auths),
|
|
144
|
-
}, flush=True)
|
|
145
|
-
except Exception:
|
|
146
|
-
pass
|
|
147
|
-
# Use 400 to make failures unmistakable during preflight
|
|
148
|
-
raise http_exception(400, "unauthorised", "API key missing or invalid", extra={
|
|
149
|
-
"allowed_first15": [k[:15] for k in allowed],
|
|
150
|
-
"allowed_count": len(allowed),
|
|
151
|
-
"got_first15": [c[:15] for c in candidates],
|
|
152
|
-
"got_lens": [len(c) for c in candidates],
|
|
153
|
-
})
|
|
164
|
+
},
|
|
165
|
+
)
|
synth_ai/task/client.py
CHANGED
|
@@ -158,7 +158,9 @@ class _TaskAppEnvironmentClient:
|
|
|
158
158
|
)
|
|
159
159
|
return response.json()
|
|
160
160
|
|
|
161
|
-
async def terminate(
|
|
161
|
+
async def terminate(
|
|
162
|
+
self, env_name: str, payload: Dict[str, Any] | None = None
|
|
163
|
+
) -> Dict[str, Any]:
|
|
162
164
|
response = await self._client._request(
|
|
163
165
|
"POST", f"/env/{env_name}/terminate", json_payload=payload or {}
|
|
164
166
|
)
|
synth_ai/task/contracts.py
CHANGED
synth_ai/task/datasets.py
CHANGED
|
@@ -37,7 +37,9 @@ class TaskDatasetRegistry:
|
|
|
37
37
|
self._entries: Dict[str, Tuple[TaskDatasetSpec, RegistryLoader, bool]] = {}
|
|
38
38
|
self._cache: Dict[Hashable, Any] = {}
|
|
39
39
|
|
|
40
|
-
def register(
|
|
40
|
+
def register(
|
|
41
|
+
self, spec: TaskDatasetSpec, loader: RegistryLoader, *, cache: bool = True
|
|
42
|
+
) -> None:
|
|
41
43
|
"""Register a dataset loader and its metadata."""
|
|
42
44
|
|
|
43
45
|
self._entries[spec.id] = (spec, loader, cache)
|
synth_ai/task/errors.py
CHANGED
|
@@ -7,7 +7,9 @@ from typing import Any, Dict, Optional
|
|
|
7
7
|
from .json import to_jsonable
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def error_payload(
|
|
10
|
+
def error_payload(
|
|
11
|
+
code: str, message: str, *, extra: Optional[Dict[str, Any]] = None
|
|
12
|
+
) -> Dict[str, Any]:
|
|
11
13
|
payload: Dict[str, Any] = {"error": {"code": code, "message": message}}
|
|
12
14
|
if extra:
|
|
13
15
|
payload["error"].update(extra)
|
|
@@ -46,4 +48,3 @@ def json_error_response(
|
|
|
46
48
|
|
|
47
49
|
payload = error_payload(code, message, extra=extra)
|
|
48
50
|
return JSONResponse(status_code=status_code, content=to_jsonable(payload), headers=headers)
|
|
49
|
-
|
synth_ai/task/health.py
CHANGED
synth_ai/task/json.py
CHANGED