synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.9.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synth-ai might be problematic. Click here for more details.
- examples/common_old/backend.py +0 -1
- examples/crafter_debug_render.py +15 -6
- examples/evals_old/compare_models.py +1 -0
- examples/finetuning_old/_backup_synth_qwen/filter_traces_achievements.py +6 -2
- examples/finetuning_old/_backup_synth_qwen/react_agent_lm.py +4 -4
- examples/finetuning_old/_backup_synth_qwen/sft_kickoff.py +4 -3
- examples/finetuning_old/synth_qwen_v1/filter_traces_achievements.py +6 -2
- examples/finetuning_old/synth_qwen_v1/finetune.py +1 -1
- examples/finetuning_old/synth_qwen_v1/hello_ft_model.py +4 -4
- examples/finetuning_old/synth_qwen_v1/infer.py +1 -2
- examples/finetuning_old/synth_qwen_v1/poll.py +4 -2
- examples/finetuning_old/synth_qwen_v1/prepare_data.py +8 -8
- examples/finetuning_old/synth_qwen_v1/react_agent_lm.py +5 -4
- examples/finetuning_old/synth_qwen_v1/run_crafter_sft_job.py +11 -8
- examples/finetuning_old/synth_qwen_v1/run_ft_job.py +17 -12
- examples/finetuning_old/synth_qwen_v1/upload_data.py +1 -1
- examples/finetuning_old/synth_qwen_v1/util.py +7 -2
- examples/rl/configs/eval_base_qwen.toml +1 -1
- examples/rl/configs/rl_from_base_qwen17.toml +1 -1
- examples/rl/download_dataset.py +26 -10
- examples/rl/run_eval.py +17 -15
- examples/rl/run_rl_and_save.py +24 -7
- examples/rl/task_app/math_single_step.py +128 -11
- examples/rl/task_app/math_task_app.py +11 -3
- examples/rl_old/task_app.py +222 -53
- examples/warming_up_to_rl/analyze_trace_db.py +7 -5
- examples/warming_up_to_rl/export_trace_sft.py +141 -16
- examples/warming_up_to_rl/groq_test.py +11 -4
- examples/warming_up_to_rl/manage_secrets.py +15 -6
- examples/warming_up_to_rl/readme.md +9 -2
- examples/warming_up_to_rl/run_eval.py +108 -30
- examples/warming_up_to_rl/run_fft_and_save.py +128 -52
- examples/warming_up_to_rl/run_local_rollout.py +87 -36
- examples/warming_up_to_rl/run_local_rollout_modal.py +113 -25
- examples/warming_up_to_rl/run_local_rollout_parallel.py +80 -16
- examples/warming_up_to_rl/run_local_rollout_traced.py +125 -20
- examples/warming_up_to_rl/run_rl_and_save.py +31 -7
- examples/warming_up_to_rl/run_rollout_remote.py +37 -10
- examples/warming_up_to_rl/task_app/grpo_crafter.py +90 -27
- examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py +9 -27
- examples/warming_up_to_rl/task_app/synth_envs_hosted/environment_routes.py +46 -108
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/app.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py +50 -17
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py +35 -21
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/react_agent.py +8 -4
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/shared.py +29 -26
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/tools.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py +17 -13
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py +106 -63
- examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py +82 -84
- examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py +76 -59
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/__init__.py +1 -1
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/volume.py +43 -49
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_service.py +5 -15
- synth_ai/__init__.py +1 -0
- synth_ai/api/train/builders.py +34 -10
- synth_ai/api/train/cli.py +172 -32
- synth_ai/api/train/config_finder.py +59 -4
- synth_ai/api/train/env_resolver.py +32 -14
- synth_ai/api/train/pollers.py +11 -3
- synth_ai/api/train/task_app.py +4 -1
- synth_ai/api/train/utils.py +20 -4
- synth_ai/cli/__init__.py +11 -4
- synth_ai/cli/balance.py +1 -1
- synth_ai/cli/demo.py +19 -5
- synth_ai/cli/rl_demo.py +75 -16
- synth_ai/cli/root.py +116 -37
- synth_ai/cli/task_apps.py +1276 -186
- synth_ai/cli/traces.py +1 -0
- synth_ai/cli/turso.py +73 -0
- synth_ai/core/experiment.py +0 -2
- synth_ai/demo_registry.py +67 -30
- synth_ai/demos/core/cli.py +493 -164
- synth_ai/demos/demo_task_apps/core.py +50 -6
- synth_ai/demos/demo_task_apps/crafter/configs/crafter_fft_4b.toml +2 -3
- synth_ai/demos/demo_task_apps/crafter/grpo_crafter_task_app.py +36 -28
- synth_ai/demos/demo_task_apps/math/_common.py +1 -2
- synth_ai/demos/demo_task_apps/math/deploy_modal.py +0 -2
- synth_ai/demos/demo_task_apps/math/modal_task_app.py +168 -65
- synth_ai/demos/demo_task_apps/math/task_app_entry.py +0 -1
- synth_ai/environments/examples/bandit/engine.py +12 -4
- synth_ai/environments/examples/bandit/taskset.py +4 -4
- synth_ai/environments/reproducibility/tree.py +3 -1
- synth_ai/environments/service/core_routes.py +6 -2
- synth_ai/evals/base.py +0 -2
- synth_ai/experimental/synth_oss.py +11 -12
- synth_ai/handshake.py +3 -1
- synth_ai/http_client.py +31 -7
- synth_ai/inference/__init__.py +0 -2
- synth_ai/inference/client.py +8 -4
- synth_ai/jobs/client.py +40 -10
- synth_ai/learning/client.py +33 -8
- synth_ai/learning/config.py +0 -2
- synth_ai/learning/constants.py +0 -2
- synth_ai/learning/ft_client.py +6 -3
- synth_ai/learning/health.py +9 -2
- synth_ai/learning/jobs.py +17 -5
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +1 -3
- synth_ai/learning/prompts/random_search.py +4 -1
- synth_ai/learning/prompts/run_random_search_banking77.py +6 -1
- synth_ai/learning/rl_client.py +42 -14
- synth_ai/learning/sse.py +0 -2
- synth_ai/learning/validators.py +6 -2
- synth_ai/lm/caching/ephemeral.py +1 -3
- synth_ai/lm/core/exceptions.py +0 -2
- synth_ai/lm/core/main.py +13 -1
- synth_ai/lm/core/synth_models.py +0 -1
- synth_ai/lm/core/vendor_clients.py +4 -2
- synth_ai/lm/overrides.py +2 -2
- synth_ai/lm/vendors/core/anthropic_api.py +7 -7
- synth_ai/lm/vendors/core/openai_api.py +2 -0
- synth_ai/lm/vendors/openai_standard.py +3 -1
- synth_ai/lm/vendors/openai_standard_responses.py +6 -3
- synth_ai/lm/vendors/supported/custom_endpoint.py +1 -3
- synth_ai/lm/vendors/synth_client.py +37 -10
- synth_ai/rl/__init__.py +0 -1
- synth_ai/rl/contracts.py +0 -2
- synth_ai/rl/env_keys.py +6 -1
- synth_ai/task/__init__.py +1 -0
- synth_ai/task/apps/__init__.py +11 -11
- synth_ai/task/auth.py +29 -17
- synth_ai/task/client.py +3 -1
- synth_ai/task/contracts.py +1 -0
- synth_ai/task/datasets.py +3 -1
- synth_ai/task/errors.py +3 -2
- synth_ai/task/health.py +0 -2
- synth_ai/task/json.py +0 -1
- synth_ai/task/proxy.py +2 -5
- synth_ai/task/rubrics.py +9 -3
- synth_ai/task/server.py +31 -5
- synth_ai/task/tracing_utils.py +8 -3
- synth_ai/task/validators.py +0 -1
- synth_ai/task/vendors.py +0 -1
- synth_ai/tracing_v3/db_config.py +26 -1
- synth_ai/tracing_v3/decorators.py +1 -0
- synth_ai/tracing_v3/examples/basic_usage.py +3 -2
- synth_ai/tracing_v3/hooks.py +2 -0
- synth_ai/tracing_v3/replica_sync.py +1 -0
- synth_ai/tracing_v3/session_tracer.py +24 -3
- synth_ai/tracing_v3/storage/base.py +4 -1
- synth_ai/tracing_v3/storage/factory.py +0 -1
- synth_ai/tracing_v3/turso/manager.py +102 -38
- synth_ai/tracing_v3/turso/models.py +4 -1
- synth_ai/tracing_v3/utils.py +1 -0
- synth_ai/v0/tracing/upload.py +32 -135
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/METADATA +1 -1
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/RECORD +154 -154
- synth_ai/install_sqld.sh +0 -40
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.9.dev5.dist-info → synth_ai-0.2.9.dev7.dist-info}/top_level.txt +0 -0
|
@@ -191,7 +191,9 @@ class BanditEngine(StatefulEngine, IReproducibleEngine):
|
|
|
191
191
|
step_count=self.step_count,
|
|
192
192
|
max_steps=self.max_steps,
|
|
193
193
|
last_arm=self.last_arm,
|
|
194
|
-
last_reward=float(reward)
|
|
194
|
+
last_reward=float(reward)
|
|
195
|
+
if reward is not None
|
|
196
|
+
else (self.last_reward if self.step_count else None),
|
|
195
197
|
cumulative_reward=float(self.total_reward),
|
|
196
198
|
reward_history=self.reward_history.copy(),
|
|
197
199
|
arm_pull_counts=self.arm_pull_counts.copy(),
|
|
@@ -238,7 +240,9 @@ class BanditEngine(StatefulEngine, IReproducibleEngine):
|
|
|
238
240
|
engine.arm_probabilities = data.get("arm_probabilities", engine.arm_probabilities)
|
|
239
241
|
engine.arm_means = data.get("arm_means", engine.arm_means)
|
|
240
242
|
engine.arm_stds = data.get("arm_stds", engine.arm_stds)
|
|
241
|
-
engine.true_expected_rewards = list(
|
|
243
|
+
engine.true_expected_rewards = list(
|
|
244
|
+
data.get("true_expected_rewards", engine.true_expected_rewards)
|
|
245
|
+
)
|
|
242
246
|
engine.arm_count = len(engine.true_expected_rewards)
|
|
243
247
|
|
|
244
248
|
engine.step_count = int(data.get("step_count", 0))
|
|
@@ -247,7 +251,9 @@ class BanditEngine(StatefulEngine, IReproducibleEngine):
|
|
|
247
251
|
engine.last_arm = data.get("last_arm")
|
|
248
252
|
engine.reward_history = list(data.get("reward_history", []))
|
|
249
253
|
engine.arm_history = list(data.get("arm_history", []))
|
|
250
|
-
engine.arm_pull_counts = list(
|
|
254
|
+
engine.arm_pull_counts = list(
|
|
255
|
+
data.get("arm_pull_counts", [0 for _ in range(engine.arm_count)])
|
|
256
|
+
)
|
|
251
257
|
engine.terminated = bool(data.get("terminated", False))
|
|
252
258
|
engine.status = data.get("status", "in_progress")
|
|
253
259
|
|
|
@@ -287,7 +293,9 @@ class SynthBanditCheckpointObservationCallable(GetObservationCallable):
|
|
|
287
293
|
"arm_count": pub.arm_count,
|
|
288
294
|
"total_reward": priv.total_reward,
|
|
289
295
|
"steps_taken": pub.step_count,
|
|
290
|
-
"best_expected_reward": max(priv.true_expected_rewards)
|
|
296
|
+
"best_expected_reward": max(priv.true_expected_rewards)
|
|
297
|
+
if priv.true_expected_rewards
|
|
298
|
+
else None,
|
|
291
299
|
"terminated": pub.terminated,
|
|
292
300
|
"status": pub.status,
|
|
293
301
|
}
|
|
@@ -156,10 +156,10 @@ async def create_bandit_taskset(
|
|
|
156
156
|
)
|
|
157
157
|
|
|
158
158
|
expected = _expected_rewards(metadata)
|
|
159
|
-
arm_count =
|
|
160
|
-
len(
|
|
161
|
-
|
|
162
|
-
or 0
|
|
159
|
+
arm_count = (
|
|
160
|
+
len(expected)
|
|
161
|
+
if expected
|
|
162
|
+
else (len(metadata.arm_probabilities or []) or len(metadata.arm_means or []) or 0)
|
|
163
163
|
)
|
|
164
164
|
if arm_count == 0:
|
|
165
165
|
arm_count = 1
|
|
@@ -256,7 +256,9 @@ class TrajectoryTreeStore:
|
|
|
256
256
|
def reconstruct_actions(self, snap_id: str) -> tuple[Any, ...]:
|
|
257
257
|
"""Return the sequence of *actions* from the root → `snap_id`."""
|
|
258
258
|
actions = []
|
|
259
|
-
for child, parent in zip(
|
|
259
|
+
for child, parent in zip(
|
|
260
|
+
self.path_to_root(snap_id)[:-1], self.path_to_root(snap_id)[1:], strict=False
|
|
261
|
+
):
|
|
260
262
|
actions.append(self.graph.edges[parent, child]["action"])
|
|
261
263
|
return tuple(reversed(actions))
|
|
262
264
|
|
|
@@ -951,7 +951,9 @@ async def register_environment_api(request: RegisterEnvironmentRequest) -> dict[
|
|
|
951
951
|
) from e
|
|
952
952
|
except Exception as e:
|
|
953
953
|
logger.error(f"Failed to register environment {request.name}: {e}")
|
|
954
|
-
raise HTTPException(
|
|
954
|
+
raise HTTPException(
|
|
955
|
+
status_code=500, detail=f"Failed to register environment: {str(e)}"
|
|
956
|
+
) from e
|
|
955
957
|
|
|
956
958
|
|
|
957
959
|
@api_router.delete("/registry/environments/{env_name}")
|
|
@@ -984,7 +986,9 @@ async def unregister_environment_api(env_name: str) -> dict[str, Any]:
|
|
|
984
986
|
|
|
985
987
|
except Exception as e:
|
|
986
988
|
logger.error(f"Failed to unregister environment {env_name}: {e}")
|
|
987
|
-
raise HTTPException(
|
|
989
|
+
raise HTTPException(
|
|
990
|
+
status_code=500, detail=f"Failed to unregister environment: {str(e)}"
|
|
991
|
+
) from e
|
|
988
992
|
|
|
989
993
|
|
|
990
994
|
@api_router.get("/registry/environments")
|
synth_ai/evals/base.py
CHANGED
|
@@ -14,7 +14,7 @@ SYNTH_BACKEND_URL = ""
|
|
|
14
14
|
# Learning V2 Modal Service URLs
|
|
15
15
|
LEARNING_V2_URLS = {
|
|
16
16
|
"dev": "https://synth-laboratories-dev--learning-v2-service-fastapi-app.modal.run",
|
|
17
|
-
"prod": "https://synth-laboratories-prod--learning-v2-service-fastapi-app.modal.run",
|
|
17
|
+
"prod": "https://synth-laboratories-prod--learning-v2-service-fastapi-app.modal.run",
|
|
18
18
|
"main": "https://synth-laboratories--learning-v2-service-fastapi-app.modal.run"
|
|
19
19
|
}
|
|
20
20
|
|
|
@@ -30,7 +30,7 @@ HEALTH_APIS = {
|
|
|
30
30
|
"response": {"status": "healthy"}
|
|
31
31
|
},
|
|
32
32
|
"detailed_health": {
|
|
33
|
-
"method": "GET",
|
|
33
|
+
"method": "GET",
|
|
34
34
|
"endpoint": "/learning/health",
|
|
35
35
|
"description": "Detailed health check including GPU function availability",
|
|
36
36
|
"response": {"status": "healthy", "components": {...}}
|
|
@@ -49,7 +49,7 @@ FILE_MANAGEMENT_APIS = {
|
|
|
49
49
|
"request": "multipart/form-data with 'file' and 'purpose'='fine-tune'",
|
|
50
50
|
"response": {
|
|
51
51
|
"id": "file-abc123",
|
|
52
|
-
"object": "file",
|
|
52
|
+
"object": "file",
|
|
53
53
|
"bytes": 1234,
|
|
54
54
|
"created_at": 1638360000,
|
|
55
55
|
"filename": "data.jsonl",
|
|
@@ -84,7 +84,7 @@ FILE_MANAGEMENT_APIS = {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
# ============================================================================
|
|
87
|
-
# TRAINING/FINE-TUNING APIS
|
|
87
|
+
# TRAINING/FINE-TUNING APIS
|
|
88
88
|
# ============================================================================
|
|
89
89
|
|
|
90
90
|
TRAINING_APIS = {
|
|
@@ -94,7 +94,7 @@ TRAINING_APIS = {
|
|
|
94
94
|
"description": "Create a fine-tuning job",
|
|
95
95
|
"request": {
|
|
96
96
|
"model": "Qwen/Qwen3-0.5B",
|
|
97
|
-
"training_file": "file-abc123",
|
|
97
|
+
"training_file": "file-abc123",
|
|
98
98
|
"training_type": "sft", # or "dpo"
|
|
99
99
|
"hyperparameters": {...},
|
|
100
100
|
"suffix": "optional"
|
|
@@ -110,7 +110,7 @@ TRAINING_APIS = {
|
|
|
110
110
|
},
|
|
111
111
|
"list_training_jobs": {
|
|
112
112
|
"method": "GET",
|
|
113
|
-
"endpoint": "/fine_tuning/jobs",
|
|
113
|
+
"endpoint": "/fine_tuning/jobs",
|
|
114
114
|
"description": "List all training jobs",
|
|
115
115
|
"response": {"object": "list", "data": ["job_objects"]}
|
|
116
116
|
},
|
|
@@ -132,7 +132,7 @@ TRAINING_APIS = {
|
|
|
132
132
|
"response": {"object": "fine_tuning.job", "id": "...", "status": "cancelled"}
|
|
133
133
|
},
|
|
134
134
|
"get_training_events": {
|
|
135
|
-
"method": "GET",
|
|
135
|
+
"method": "GET",
|
|
136
136
|
"endpoint": "/fine_tuning/jobs/{job_id}/events",
|
|
137
137
|
"description": "Get training logs/events",
|
|
138
138
|
"response": {
|
|
@@ -154,7 +154,7 @@ TRAINING_APIS = {
|
|
|
154
154
|
INFERENCE_APIS = {
|
|
155
155
|
"chat_completions": {
|
|
156
156
|
"method": "POST",
|
|
157
|
-
"endpoint": "/chat/completions",
|
|
157
|
+
"endpoint": "/chat/completions",
|
|
158
158
|
"description": "OpenAI-compatible chat completions for base and fine-tuned models",
|
|
159
159
|
"request": {
|
|
160
160
|
"model": "Qwen/Qwen3-0.5B", # or "ft:Qwen/Qwen3-0.5B:suffix"
|
|
@@ -190,7 +190,7 @@ INFERENCE_APIS = {
|
|
|
190
190
|
}
|
|
191
191
|
}
|
|
192
192
|
|
|
193
|
-
# ============================================================================
|
|
193
|
+
# ============================================================================
|
|
194
194
|
# MODEL MANAGEMENT APIS
|
|
195
195
|
# ============================================================================
|
|
196
196
|
|
|
@@ -203,7 +203,7 @@ MODEL_APIS = {
|
|
|
203
203
|
"object": "list",
|
|
204
204
|
"data": [{
|
|
205
205
|
"id": "Qwen/Qwen3-0.5B",
|
|
206
|
-
"object": "model",
|
|
206
|
+
"object": "model",
|
|
207
207
|
"created": 1638360000,
|
|
208
208
|
"owned_by": "learning_v2"
|
|
209
209
|
}]
|
|
@@ -246,7 +246,7 @@ SUPPORTED_MODELS = {
|
|
|
246
246
|
"gpu_types": ["A10G", "L40S", "A100", "H100"],
|
|
247
247
|
"features": [
|
|
248
248
|
"Tool calling",
|
|
249
|
-
"Streaming responses",
|
|
249
|
+
"Streaming responses",
|
|
250
250
|
"Fine-tuning",
|
|
251
251
|
"Multi-GPU training",
|
|
252
252
|
"JSONL data format",
|
|
@@ -338,7 +338,6 @@ working.
|
|
|
338
338
|
|
|
339
339
|
'''
|
|
340
340
|
|
|
341
|
-
|
|
342
341
|
"""
|
|
343
342
|
LEARNING_v2 server-side changes required to honor `X-GPU-Preference`
|
|
344
343
|
====================================================================
|
synth_ai/handshake.py
CHANGED
|
@@ -72,7 +72,9 @@ def start_handshake_session(origin: str | None = None) -> Tuple[str, str, int, i
|
|
|
72
72
|
)
|
|
73
73
|
|
|
74
74
|
|
|
75
|
-
def poll_handshake_token(
|
|
75
|
+
def poll_handshake_token(
|
|
76
|
+
device_code: str, origin: str | None = None, *, timeout_s: int | None = None
|
|
77
|
+
) -> Dict[str, Any]:
|
|
76
78
|
base = (origin or _origin()).rstrip("/")
|
|
77
79
|
api_origin, _ = _split_origin(base)
|
|
78
80
|
url = urljoin(api_origin.rstrip("/") + "/", "api/sdk/handshake/token")
|
synth_ai/http_client.py
CHANGED
|
@@ -48,26 +48,46 @@ class AsyncHttpClient:
|
|
|
48
48
|
path = path[4:] # Remove leading /api
|
|
49
49
|
return f"{self._base_url}/{path.lstrip('/')}"
|
|
50
50
|
|
|
51
|
-
async def get(
|
|
51
|
+
async def get(
|
|
52
|
+
self,
|
|
53
|
+
path: str,
|
|
54
|
+
*,
|
|
55
|
+
params: Optional[Dict[str, Any]] = None,
|
|
56
|
+
headers: Optional[Dict[str, str]] = None,
|
|
57
|
+
) -> Any:
|
|
52
58
|
url = self._abs(path)
|
|
53
59
|
assert self._session is not None, "AsyncHttpClient must be used as an async context manager"
|
|
54
60
|
async with self._session.get(url, params=params, headers=headers) as resp:
|
|
55
61
|
return await self._handle_response(resp, url)
|
|
56
62
|
|
|
57
|
-
async def post_json(
|
|
63
|
+
async def post_json(
|
|
64
|
+
self, path: str, *, json: Dict[str, Any], headers: Optional[Dict[str, str]] = None
|
|
65
|
+
) -> Any:
|
|
58
66
|
url = self._abs(path)
|
|
59
67
|
assert self._session is not None, "AsyncHttpClient must be used as an async context manager"
|
|
60
68
|
async with self._session.post(url, json=json, headers=headers) as resp:
|
|
61
69
|
return await self._handle_response(resp, url)
|
|
62
70
|
|
|
63
|
-
async def post_multipart(
|
|
71
|
+
async def post_multipart(
|
|
72
|
+
self,
|
|
73
|
+
path: str,
|
|
74
|
+
*,
|
|
75
|
+
data: Dict[str, Any],
|
|
76
|
+
files: Dict[str, tuple[str, bytes, str | None]],
|
|
77
|
+
headers: Optional[Dict[str, str]] = None,
|
|
78
|
+
) -> Any:
|
|
64
79
|
url = self._abs(path)
|
|
65
80
|
assert self._session is not None, "AsyncHttpClient must be used as an async context manager"
|
|
66
81
|
form = aiohttp.FormData()
|
|
67
82
|
for k, v in data.items():
|
|
68
83
|
form.add_field(k, str(v))
|
|
69
84
|
for field, (filename, content, content_type) in files.items():
|
|
70
|
-
form.add_field(
|
|
85
|
+
form.add_field(
|
|
86
|
+
field,
|
|
87
|
+
content,
|
|
88
|
+
filename=filename,
|
|
89
|
+
content_type=content_type or "application/octet-stream",
|
|
90
|
+
)
|
|
71
91
|
async with self._session.post(url, data=form, headers=headers) as resp:
|
|
72
92
|
return await self._handle_response(resp, url)
|
|
73
93
|
|
|
@@ -95,10 +115,14 @@ class AsyncHttpClient:
|
|
|
95
115
|
detail = await resp.json()
|
|
96
116
|
except Exception:
|
|
97
117
|
detail = None
|
|
98
|
-
raise HTTPError(
|
|
118
|
+
raise HTTPError(
|
|
119
|
+
status=resp.status,
|
|
120
|
+
url=url,
|
|
121
|
+
message="request_failed",
|
|
122
|
+
body_snippet=body_snippet,
|
|
123
|
+
detail=detail,
|
|
124
|
+
)
|
|
99
125
|
|
|
100
126
|
|
|
101
127
|
async def sleep(seconds: float) -> None:
|
|
102
128
|
await asyncio.sleep(seconds)
|
|
103
|
-
|
|
104
|
-
|
synth_ai/inference/__init__.py
CHANGED
synth_ai/inference/client.py
CHANGED
|
@@ -11,10 +11,14 @@ class InferenceClient:
|
|
|
11
11
|
self._api_key = api_key
|
|
12
12
|
self._timeout = timeout
|
|
13
13
|
|
|
14
|
-
async def create_chat_completion(
|
|
14
|
+
async def create_chat_completion(
|
|
15
|
+
self, *, model: str, messages: list[dict], **kwargs: Any
|
|
16
|
+
) -> Dict[str, Any]:
|
|
15
17
|
body: Dict[str, Any] = {"model": model, "messages": messages}
|
|
16
18
|
body.update(kwargs)
|
|
19
|
+
# Backend now expects an explicit thinking_budget; provide a sensible default if omitted
|
|
20
|
+
if "thinking_budget" not in body:
|
|
21
|
+
body["thinking_budget"] = 256
|
|
17
22
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
23
|
+
# Public learning-v2 inference path mounted under /api/v1
|
|
24
|
+
return await http.post_json("/api/v1/chat/completions", json=body)
|
synth_ai/jobs/client.py
CHANGED
|
@@ -9,13 +9,25 @@ class FilesApi:
|
|
|
9
9
|
def __init__(self, http: AsyncHttpClient) -> None:
|
|
10
10
|
self._http = http
|
|
11
11
|
|
|
12
|
-
async def upload(
|
|
12
|
+
async def upload(
|
|
13
|
+
self,
|
|
14
|
+
*,
|
|
15
|
+
filename: str,
|
|
16
|
+
content: bytes,
|
|
17
|
+
purpose: str,
|
|
18
|
+
content_type: Optional[str] = None,
|
|
19
|
+
idempotency_key: Optional[str] = None,
|
|
20
|
+
) -> Dict[str, Any]:
|
|
13
21
|
data = {"purpose": purpose}
|
|
14
22
|
files = {"file": (filename, content, content_type)}
|
|
15
23
|
headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None
|
|
16
|
-
return await self._http.post_multipart(
|
|
24
|
+
return await self._http.post_multipart(
|
|
25
|
+
"/api/files", data=data, files=files, headers=headers
|
|
26
|
+
)
|
|
17
27
|
|
|
18
|
-
async def list(
|
|
28
|
+
async def list(
|
|
29
|
+
self, *, purpose: Optional[str] = None, after: Optional[str] = None, limit: int = 20
|
|
30
|
+
) -> Dict[str, Any]:
|
|
19
31
|
params: Dict[str, Any] = {}
|
|
20
32
|
if purpose is not None:
|
|
21
33
|
params["purpose"] = purpose
|
|
@@ -30,7 +42,9 @@ class FilesApi:
|
|
|
30
42
|
async def delete(self, file_id: str) -> Any:
|
|
31
43
|
return await self._http.delete(f"/api/files/{file_id}")
|
|
32
44
|
|
|
33
|
-
async def list_jobs(
|
|
45
|
+
async def list_jobs(
|
|
46
|
+
self, file_id: str, *, after: Optional[str] = None, limit: int = 20
|
|
47
|
+
) -> Dict[str, Any]:
|
|
34
48
|
params: Dict[str, Any] = {"limit": limit}
|
|
35
49
|
if after is not None:
|
|
36
50
|
params["after"] = after
|
|
@@ -102,11 +116,15 @@ class SftJobsApi:
|
|
|
102
116
|
async def cancel(self, job_id: str) -> Dict[str, Any]:
|
|
103
117
|
return await self._http.post_json(f"/api/sft/jobs/{job_id}/cancel", json={})
|
|
104
118
|
|
|
105
|
-
async def list_events(
|
|
119
|
+
async def list_events(
|
|
120
|
+
self, job_id: str, *, since_seq: int = 0, limit: int = 200
|
|
121
|
+
) -> Dict[str, Any]:
|
|
106
122
|
params = {"since_seq": since_seq, "limit": limit}
|
|
107
123
|
return await self._http.get(f"/api/sft/jobs/{job_id}/events", params=params)
|
|
108
124
|
|
|
109
|
-
async def checkpoints(
|
|
125
|
+
async def checkpoints(
|
|
126
|
+
self, job_id: str, *, after: Optional[str] = None, limit: int = 10
|
|
127
|
+
) -> Dict[str, Any]:
|
|
110
128
|
params: Dict[str, Any] = {"limit": limit}
|
|
111
129
|
if after is not None:
|
|
112
130
|
params["after"] = after
|
|
@@ -174,11 +192,15 @@ class RlJobsApi:
|
|
|
174
192
|
async def cancel(self, job_id: str) -> Dict[str, Any]:
|
|
175
193
|
return await self._http.post_json(f"/api/rl/jobs/{job_id}/cancel", json={})
|
|
176
194
|
|
|
177
|
-
async def list_events(
|
|
195
|
+
async def list_events(
|
|
196
|
+
self, job_id: str, *, since_seq: int = 0, limit: int = 200
|
|
197
|
+
) -> Dict[str, Any]:
|
|
178
198
|
params = {"since_seq": since_seq, "limit": limit}
|
|
179
199
|
return await self._http.get(f"/api/rl/jobs/{job_id}/events", params=params)
|
|
180
200
|
|
|
181
|
-
async def metrics(
|
|
201
|
+
async def metrics(
|
|
202
|
+
self, job_id: str, *, after_step: int = -1, limit: int = 200
|
|
203
|
+
) -> Dict[str, Any]:
|
|
182
204
|
params = {"after_step": after_step, "limit": limit}
|
|
183
205
|
return await self._http.get(f"/api/rl/jobs/{job_id}/metrics", params=params)
|
|
184
206
|
|
|
@@ -213,7 +235,9 @@ class ModelsApi:
|
|
|
213
235
|
async def delete(self, model_id: str) -> Any:
|
|
214
236
|
return await self._http.delete(f"/api/models/{model_id}")
|
|
215
237
|
|
|
216
|
-
async def list_jobs(
|
|
238
|
+
async def list_jobs(
|
|
239
|
+
self, model_id: str, *, after: Optional[str] = None, limit: int = 20
|
|
240
|
+
) -> Dict[str, Any]:
|
|
217
241
|
params: Dict[str, Any] = {"limit": limit}
|
|
218
242
|
if after is not None:
|
|
219
243
|
params["after"] = after
|
|
@@ -228,7 +252,13 @@ class JobsClient:
|
|
|
228
252
|
await c.files.list()
|
|
229
253
|
"""
|
|
230
254
|
|
|
231
|
-
def __init__(
|
|
255
|
+
def __init__(
|
|
256
|
+
self,
|
|
257
|
+
base_url: str,
|
|
258
|
+
api_key: str,
|
|
259
|
+
timeout: float = 30.0,
|
|
260
|
+
http: Optional[AsyncHttpClient] = None,
|
|
261
|
+
) -> None:
|
|
232
262
|
self._base_url = base_url
|
|
233
263
|
self._api_key = api_key
|
|
234
264
|
self._timeout = timeout
|
synth_ai/learning/client.py
CHANGED
|
@@ -20,7 +20,12 @@ class LearningClient:
|
|
|
20
20
|
files = {"file": (p.name, content, _infer_content_type(p.name))}
|
|
21
21
|
js = await http.post_multipart("/api/learning/files", data=data, files=files)
|
|
22
22
|
if not isinstance(js, dict) or "id" not in js:
|
|
23
|
-
raise HTTPError(
|
|
23
|
+
raise HTTPError(
|
|
24
|
+
status=500,
|
|
25
|
+
url="/api/learning/files",
|
|
26
|
+
message="invalid_upload_response",
|
|
27
|
+
body_snippet=str(js)[:200],
|
|
28
|
+
)
|
|
24
29
|
return str(js["id"])
|
|
25
30
|
|
|
26
31
|
async def create_job(
|
|
@@ -50,7 +55,9 @@ class LearningClient:
|
|
|
50
55
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
|
|
51
56
|
return await http.get(f"/api/learning/jobs/{job_id}")
|
|
52
57
|
|
|
53
|
-
async def get_events(
|
|
58
|
+
async def get_events(
|
|
59
|
+
self, job_id: str, *, since_seq: int = 0, limit: int = 200
|
|
60
|
+
) -> List[Dict[str, Any]]:
|
|
54
61
|
params = {"since_seq": since_seq, "limit": limit}
|
|
55
62
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
|
|
56
63
|
js = await http.get(f"/api/learning/jobs/{job_id}/events", params=params)
|
|
@@ -58,7 +65,15 @@ class LearningClient:
|
|
|
58
65
|
return js["events"]
|
|
59
66
|
return []
|
|
60
67
|
|
|
61
|
-
async def get_metrics(
|
|
68
|
+
async def get_metrics(
|
|
69
|
+
self,
|
|
70
|
+
job_id: str,
|
|
71
|
+
*,
|
|
72
|
+
name: str | None = None,
|
|
73
|
+
after_step: int | None = None,
|
|
74
|
+
limit: int = 500,
|
|
75
|
+
run_id: str | None = None,
|
|
76
|
+
) -> List[Dict[str, Any]]:
|
|
62
77
|
params: Dict[str, Any] = {"limit": limit}
|
|
63
78
|
if name is not None:
|
|
64
79
|
params["name"] = name
|
|
@@ -115,7 +130,9 @@ class LearningClient:
|
|
|
115
130
|
raise TimeoutError(f"Polling timed out after {elapsed} seconds for job {job_id}")
|
|
116
131
|
|
|
117
132
|
# --- Optional diagnostics ---
|
|
118
|
-
async def pricing_preflight(
|
|
133
|
+
async def pricing_preflight(
|
|
134
|
+
self, *, job_type: str, gpu_type: str, estimated_seconds: float, container_count: int
|
|
135
|
+
) -> Dict[str, Any]:
|
|
119
136
|
body = {
|
|
120
137
|
"job_type": job_type,
|
|
121
138
|
"gpu_type": gpu_type,
|
|
@@ -125,14 +142,24 @@ class LearningClient:
|
|
|
125
142
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
|
|
126
143
|
js = await http.post_json("/api/v1/pricing/preflight", json=body)
|
|
127
144
|
if not isinstance(js, dict):
|
|
128
|
-
raise HTTPError(
|
|
145
|
+
raise HTTPError(
|
|
146
|
+
status=500,
|
|
147
|
+
url="/api/v1/pricing/preflight",
|
|
148
|
+
message="invalid_preflight_response",
|
|
149
|
+
body_snippet=str(js)[:200],
|
|
150
|
+
)
|
|
129
151
|
return js
|
|
130
152
|
|
|
131
153
|
async def balance_autumn_normalized(self) -> Dict[str, Any]:
|
|
132
154
|
async with AsyncHttpClient(self._base_url, self._api_key, timeout=self._timeout) as http:
|
|
133
155
|
js = await http.get("/api/v1/balance/autumn-normalized")
|
|
134
156
|
if not isinstance(js, dict):
|
|
135
|
-
raise HTTPError(
|
|
157
|
+
raise HTTPError(
|
|
158
|
+
status=500,
|
|
159
|
+
url="/api/v1/balance/autumn-normalized",
|
|
160
|
+
message="invalid_balance_response",
|
|
161
|
+
body_snippet=str(js)[:200],
|
|
162
|
+
)
|
|
136
163
|
return js
|
|
137
164
|
|
|
138
165
|
|
|
@@ -145,5 +172,3 @@ def _infer_content_type(filename: str) -> str:
|
|
|
145
172
|
if name.endswith(".txt"):
|
|
146
173
|
return "text/plain"
|
|
147
174
|
return "application/octet-stream"
|
|
148
|
-
|
|
149
|
-
|
synth_ai/learning/config.py
CHANGED
synth_ai/learning/constants.py
CHANGED
synth_ai/learning/ft_client.py
CHANGED
|
@@ -20,7 +20,12 @@ class FtClient:
|
|
|
20
20
|
files = {"file": (p.name, content, _infer_content_type(p.name))}
|
|
21
21
|
js = await http.post_multipart("/api/learning/files", data=data, files=files)
|
|
22
22
|
if not isinstance(js, dict) or "id" not in js:
|
|
23
|
-
raise HTTPError(
|
|
23
|
+
raise HTTPError(
|
|
24
|
+
status=500,
|
|
25
|
+
url="/api/learning/files",
|
|
26
|
+
message="invalid_upload_response",
|
|
27
|
+
body_snippet=str(js)[:200],
|
|
28
|
+
)
|
|
24
29
|
return str(js["id"])
|
|
25
30
|
|
|
26
31
|
async def create_sft_job(
|
|
@@ -55,5 +60,3 @@ def _infer_content_type(filename: str) -> str:
|
|
|
55
60
|
if name.endswith(".txt"):
|
|
56
61
|
return "text/plain"
|
|
57
62
|
return "application/octet-stream"
|
|
58
|
-
|
|
59
|
-
|
synth_ai/learning/health.py
CHANGED
|
@@ -24,7 +24,15 @@ async def task_app_health(task_app_url: str) -> Dict[str, Any]:
|
|
|
24
24
|
return await _th(task_app_url)
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
async def pricing_preflight(
|
|
27
|
+
async def pricing_preflight(
|
|
28
|
+
base_url: str,
|
|
29
|
+
api_key: str,
|
|
30
|
+
*,
|
|
31
|
+
job_type: str,
|
|
32
|
+
gpu_type: str,
|
|
33
|
+
estimated_seconds: float,
|
|
34
|
+
container_count: int,
|
|
35
|
+
) -> Dict[str, Any]:
|
|
28
36
|
body = {
|
|
29
37
|
"job_type": job_type,
|
|
30
38
|
"gpu_type": gpu_type,
|
|
@@ -40,4 +48,3 @@ async def balance_autumn_normalized(base_url: str, api_key: str) -> Dict[str, An
|
|
|
40
48
|
async with AsyncHttpClient(base_url, api_key, timeout=30.0) as http:
|
|
41
49
|
js = await http.get(f"{_api_base(base_url)}/v1/balance/autumn-normalized")
|
|
42
50
|
return js if isinstance(js, dict) else {"raw": js}
|
|
43
|
-
|
synth_ai/learning/jobs.py
CHANGED
|
@@ -40,7 +40,15 @@ class JobsApiResolver:
|
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
class JobHandle:
|
|
43
|
-
def __init__(
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
base_url: str,
|
|
46
|
+
api_key: str,
|
|
47
|
+
job_id: str,
|
|
48
|
+
*,
|
|
49
|
+
strict: bool = True,
|
|
50
|
+
timeout: float = 600.0,
|
|
51
|
+
) -> None:
|
|
44
52
|
self.base_url = base_url.rstrip("/")
|
|
45
53
|
self.api_key = api_key
|
|
46
54
|
self.job_id = job_id
|
|
@@ -134,7 +142,11 @@ class JobHandle:
|
|
|
134
142
|
if not detected_fine_tuned_model:
|
|
135
143
|
try:
|
|
136
144
|
data_obj = e.get("data") or {}
|
|
137
|
-
ftm =
|
|
145
|
+
ftm = (
|
|
146
|
+
data_obj.get("fine_tuned_model")
|
|
147
|
+
if isinstance(data_obj, dict)
|
|
148
|
+
else None
|
|
149
|
+
)
|
|
138
150
|
if isinstance(ftm, str) and ftm:
|
|
139
151
|
detected_fine_tuned_model = ftm
|
|
140
152
|
except Exception:
|
|
@@ -200,6 +212,6 @@ class JobHandle:
|
|
|
200
212
|
)
|
|
201
213
|
await sleep(interval_seconds)
|
|
202
214
|
if max_seconds is not None and (time.time() - start_t) >= max_seconds:
|
|
203
|
-
raise TimeoutError(
|
|
204
|
-
|
|
205
|
-
|
|
215
|
+
raise TimeoutError(
|
|
216
|
+
f"Polling timed out after {max_seconds}s for job {self.job_id}"
|
|
217
|
+
)
|
|
@@ -171,9 +171,7 @@ async def main() -> None:
|
|
|
171
171
|
with LMOverridesContext(
|
|
172
172
|
[{"match": {"contains": "atm"}, "injection_rules": INJECTION_RULES}]
|
|
173
173
|
):
|
|
174
|
-
_ = await client.chat.completions.create(
|
|
175
|
-
model=model, messages=messages, temperature=0
|
|
176
|
-
)
|
|
174
|
+
_ = await client.chat.completions.create(model=model, messages=messages, temperature=0)
|
|
177
175
|
# Not all models echo input; instead, verify that our injected expectation matches
|
|
178
176
|
expected_user = messages[1]["content"].replace("atm", "ATM")
|
|
179
177
|
if messages[1]["content"] == expected_user:
|
|
@@ -148,7 +148,10 @@ def random_search_compile(
|
|
|
148
148
|
max_rounds: int = 2,
|
|
149
149
|
num_candidate_programs: int = 16,
|
|
150
150
|
stop_at_score: float | None = None,
|
|
151
|
-
evaluate_fn: Callable[
|
|
151
|
+
evaluate_fn: Callable[
|
|
152
|
+
[_ProgramLike, Sequence[tuple[Any, Any]], Callable[[Any, Any], float]], EvalResult
|
|
153
|
+
]
|
|
154
|
+
| None = None,
|
|
152
155
|
on_candidate_evaluated: Callable[[int, float, EvalResult, dict[str, Any]], None] | None = None,
|
|
153
156
|
) -> tuple[_ProgramLike, list[dict[str, Any]]]:
|
|
154
157
|
best_program: _ProgramLike | None = None
|
|
@@ -145,7 +145,10 @@ def main():
|
|
|
145
145
|
t_end = time.monotonic()
|
|
146
146
|
return i, y, "", t_start, t_end, {}
|
|
147
147
|
|
|
148
|
-
tasks = [
|
|
148
|
+
tasks = [
|
|
149
|
+
asyncio.create_task(worker(i, x, y))
|
|
150
|
+
for i, (x, y) in enumerate(zip(xs, ys, strict=False))
|
|
151
|
+
]
|
|
149
152
|
correct_sum = 0.0
|
|
150
153
|
processed = 0
|
|
151
154
|
import statistics
|
|
@@ -185,6 +188,7 @@ def main():
|
|
|
185
188
|
pending, timeout=timeout, return_when=asyncio.FIRST_COMPLETED
|
|
186
189
|
)
|
|
187
190
|
import contextlib
|
|
191
|
+
|
|
188
192
|
for task in done:
|
|
189
193
|
try:
|
|
190
194
|
i, y_true, pred, t_start, t_end, usage = task.result()
|
|
@@ -251,6 +255,7 @@ def main():
|
|
|
251
255
|
pbar.set_postfix({"score": f"{score:.2f}"})
|
|
252
256
|
# store per-instance details (for apples-to-apples)
|
|
253
257
|
import contextlib
|
|
258
|
+
|
|
254
259
|
with contextlib.suppress(Exception):
|
|
255
260
|
candidate_eval_details[idx] = {
|
|
256
261
|
"score": score,
|