synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- examples/README.md +1 -0
- examples/__init__.py +16 -0
- examples/analyze_semantic_words.sh +17 -0
- examples/baseline/banking77_baseline.py +243 -0
- examples/baseline/banking77_pipeline_baseline.py +294 -0
- examples/baseline/crafter_baseline.py +407 -0
- examples/baseline/pokemon_red_baseline.py +326 -0
- examples/baseline/simple_baseline.py +56 -0
- examples/baseline/warming_up_to_rl_baseline.py +239 -0
- examples/blog_posts/gepa/README.md +355 -0
- examples/blog_posts/gepa/configs/banking77_gepa_local.toml +95 -0
- examples/blog_posts/gepa/configs/banking77_gepa_test.toml +80 -0
- examples/blog_posts/gepa/configs/banking77_mipro_local.toml +50 -0
- examples/blog_posts/gepa/configs/banking77_pipeline_gepa_local.toml +101 -0
- examples/blog_posts/gepa/configs/banking77_pipeline_gepa_test.toml +96 -0
- examples/blog_posts/gepa/configs/hotpotqa_gepa_local.toml +57 -0
- examples/blog_posts/gepa/configs/hotpotqa_gepa_qwen.toml +35 -0
- examples/blog_posts/gepa/configs/hotpotqa_mipro_local.toml +51 -0
- examples/blog_posts/gepa/configs/hover_gepa_local.toml +57 -0
- examples/blog_posts/gepa/configs/hover_gepa_qwen.toml +35 -0
- examples/blog_posts/gepa/configs/hover_mipro_local.toml +51 -0
- examples/blog_posts/gepa/configs/ifbench_gepa_local.toml +57 -0
- examples/blog_posts/gepa/configs/ifbench_gepa_qwen.toml +35 -0
- examples/blog_posts/gepa/configs/ifbench_mipro_local.toml +51 -0
- examples/blog_posts/gepa/configs/pupa_gepa_local.toml +58 -0
- examples/blog_posts/gepa/configs/pupa_mipro_local.toml +52 -0
- examples/blog_posts/gepa/deploy_banking77_task_app.sh +54 -0
- examples/blog_posts/gepa/gepa_baseline.py +204 -0
- examples/blog_posts/gepa/query_prompts_example.py +97 -0
- examples/blog_posts/gepa/run_gepa_banking77.sh +112 -0
- examples/blog_posts/gepa/run_gepa_banking77_pipeline.sh +163 -0
- examples/blog_posts/gepa/task_apps.py +105 -0
- examples/blog_posts/gepa/test_gepa_local.sh +67 -0
- examples/blog_posts/gepa/verify_banking77_setup.sh +123 -0
- examples/blog_posts/mipro/README.md +415 -0
- examples/blog_posts/mipro/configs/banking77_mipro_local.toml +91 -0
- examples/blog_posts/mipro/configs/banking77_mipro_test.toml +87 -0
- examples/blog_posts/mipro/configs/banking77_pipeline_mipro_gemini_flash_lite_local.toml +98 -0
- examples/blog_posts/mipro/configs/banking77_pipeline_mipro_gpt41mini_local.toml +96 -0
- examples/blog_posts/mipro/configs/banking77_pipeline_mipro_local.toml +94 -0
- examples/blog_posts/mipro/configs/banking77_pipeline_mipro_test.toml +170 -0
- examples/blog_posts/mipro/deploy_banking77_pipeline_task_app.sh +59 -0
- examples/blog_posts/mipro/deploy_banking77_task_app.sh +41 -0
- examples/blog_posts/mipro/multi_step.md +79 -0
- examples/blog_posts/mipro/run_mipro_banking77.sh +191 -0
- examples/blog_posts/mipro/run_mipro_banking77_pipeline.sh +171 -0
- examples/blog_posts/mipro/run_mipro_banking77_pipeline_gemini_flash_lite.sh +177 -0
- examples/blog_posts/mipro/run_mipro_banking77_pipeline_gpt41mini.sh +173 -0
- examples/blog_posts/mipro/verify_banking77_setup.sh +117 -0
- examples/blog_posts/pokemon_vl/README.md +98 -0
- examples/blog_posts/pokemon_vl/configs/eval_gpt5nano.toml +26 -0
- examples/blog_posts/pokemon_vl/configs/eval_qwen3_vl.toml +27 -0
- examples/blog_posts/pokemon_vl/configs/eval_rl_final.toml +24 -0
- examples/blog_posts/pokemon_vl/configs/filter_high_reward.toml +10 -0
- examples/blog_posts/pokemon_vl/configs/train_rl_from_sft.toml +43 -0
- examples/blog_posts/pokemon_vl/configs/train_sft_qwen4b_vl.toml +40 -0
- examples/blog_posts/pokemon_vl/extract_images.py +239 -0
- examples/blog_posts/pokemon_vl/pokemon_vl_baseline.py +326 -0
- examples/blog_posts/pokemon_vl/run_eval_extract_images.py +209 -0
- examples/blog_posts/pokemon_vl/run_qwen_eval_extract_images.py +212 -0
- examples/blog_posts/pokemon_vl/text_box_analysis.md +106 -0
- examples/blog_posts/warming_up_to_rl/ARCHITECTURE.md +195 -0
- examples/blog_posts/warming_up_to_rl/FINAL_TEST_RESULTS.md +127 -0
- examples/blog_posts/warming_up_to_rl/INFERENCE_SUCCESS.md +132 -0
- examples/blog_posts/warming_up_to_rl/README.md +158 -0
- examples/blog_posts/warming_up_to_rl/SMOKE_TESTING.md +164 -0
- examples/blog_posts/warming_up_to_rl/SMOKE_TEST_COMPLETE.md +253 -0
- examples/blog_posts/warming_up_to_rl/configs/eval_baseline_qwen32b_10x20.toml +25 -0
- examples/blog_posts/warming_up_to_rl/configs/eval_ft_qwen4b.toml +25 -0
- examples/blog_posts/warming_up_to_rl/configs/eval_ft_qwen4b_10x20.toml +26 -0
- examples/blog_posts/warming_up_to_rl/configs/eval_groq_qwen32b.toml +25 -0
- examples/blog_posts/warming_up_to_rl/configs/eval_openai_gpt_oss_120b.toml +29 -0
- examples/blog_posts/warming_up_to_rl/configs/filter_high_reward_dataset.toml +10 -0
- examples/blog_posts/warming_up_to_rl/configs/smoke_test.toml +75 -0
- examples/blog_posts/warming_up_to_rl/configs/train_rl_from_sft.toml +91 -0
- examples/blog_posts/warming_up_to_rl/configs/train_sft_qwen4b.toml +40 -0
- examples/blog_posts/warming_up_to_rl/warming_up_to_rl_baseline.py +187 -0
- examples/crafter_debug_render.py +186 -0
- examples/dev/qwen3_32b_qlora_4xh100.toml +45 -0
- examples/gepa/banking77_pipeline_gepa.toml +96 -0
- examples/gepa/multi_stage_gepa_example.toml +84 -0
- examples/gepa/run_gepa_banking77_pipeline.sh +157 -0
- examples/multi_step/SFT_README.md +147 -0
- examples/multi_step/configs/README_verilog_rl.md +77 -0
- examples/multi_step/configs/VERILOG_REWARDS.md +103 -0
- examples/multi_step/configs/VERILOG_RL_CHECKLIST.md +196 -0
- examples/multi_step/configs/crafter_eval_synth_qwen4b.toml +35 -0
- examples/multi_step/configs/crafter_eval_text_only_groq_qwen32b.toml +36 -0
- examples/multi_step/configs/crafter_rl_outcome.toml +75 -0
- examples/multi_step/configs/crafter_rl_stepwise_hosted_judge.toml +145 -0
- examples/multi_step/configs/crafter_rl_stepwise_shaped.toml +84 -0
- examples/multi_step/configs/crafter_rl_stepwise_simple.toml +79 -0
- examples/multi_step/configs/crafter_rl_stepwise_simple_NEW_FORMAT.toml +105 -0
- examples/multi_step/configs/crafter_sft_qwen30b_lora.toml +62 -0
- examples/multi_step/configs/crafter_synth_backend.md +40 -0
- examples/multi_step/configs/verilog_eval_groq_qwen32b.toml +31 -0
- examples/multi_step/configs/verilog_eval_synth_qwen8b.toml +33 -0
- examples/multi_step/configs/verilog_rl_lora.toml +147 -0
- examples/multi_step/convert_traces_to_sft.py +84 -0
- examples/multi_step/crafter_rl_lora.md +70 -0
- examples/multi_step/judges/crafter_backend_judge.py +220 -0
- examples/multi_step/judges/verilog_backend_judge.py +234 -0
- examples/multi_step/readme.md +48 -0
- examples/multi_step/run_sft_qwen30b.sh +45 -0
- examples/multi_step/sse_metrics_streaming_notes.md +357 -0
- examples/multi_step/task_app_config_notes.md +494 -0
- examples/multi_step/verilog_rl_lora.md +218 -0
- examples/qwen_coder/README.md +102 -0
- examples/qwen_coder/_shared.py +113 -0
- examples/qwen_coder/configs/coder_lora_30b.toml +60 -0
- examples/qwen_coder/configs/coder_lora_4b.toml +61 -0
- examples/qwen_coder/configs/coder_lora_small.toml +57 -0
- examples/qwen_coder/generate_dataset.py +98 -0
- examples/qwen_coder/infer_ft_smoke.py +65 -0
- examples/qwen_coder/infer_prod_proxy.py +73 -0
- examples/qwen_coder/infer_via_synth.py +87 -0
- examples/qwen_coder/scripts/infer_coder.sh +19 -0
- examples/qwen_coder/scripts/train_coder_30b.sh +22 -0
- examples/qwen_coder/sft_full_17b.py +103 -0
- examples/qwen_coder/sft_lora_30b.py +110 -0
- examples/qwen_coder/subset_jsonl.py +39 -0
- examples/qwen_coder/todos.md +38 -0
- examples/qwen_coder/validate_jsonl.py +60 -0
- examples/qwen_vl/BUGS_AND_FIXES.md +232 -0
- examples/qwen_vl/IMAGE_VALIDATION_COMPLETE.md +271 -0
- examples/qwen_vl/IMAGE_VALIDATION_SUMMARY.md +260 -0
- examples/qwen_vl/INFERENCE_SFT_TESTS.md +412 -0
- examples/qwen_vl/NEXT_STEPS_2B.md +325 -0
- examples/qwen_vl/QUICKSTART.md +327 -0
- examples/qwen_vl/QUICKSTART_RL_VISION.md +110 -0
- examples/qwen_vl/README.md +152 -0
- examples/qwen_vl/RL_VISION_COMPLETE.md +475 -0
- examples/qwen_vl/RL_VISION_TESTING.md +333 -0
- examples/qwen_vl/SDK_VISION_INTEGRATION.md +328 -0
- examples/qwen_vl/SETUP_COMPLETE.md +274 -0
- examples/qwen_vl/VISION_TESTS_COMPLETE.md +489 -0
- examples/qwen_vl/VLM_PIPELINE_COMPLETE.md +242 -0
- examples/qwen_vl/__init__.py +2 -0
- examples/qwen_vl/collect_data_via_cli.md +415 -0
- examples/qwen_vl/collect_vision_traces.py +368 -0
- examples/qwen_vl/configs/crafter_rl_vision_qwen3vl4b.toml +110 -0
- examples/qwen_vl/configs/crafter_vlm_sft_example.toml +59 -0
- examples/qwen_vl/configs/eval_gpt4o_mini_vision.toml +26 -0
- examples/qwen_vl/configs/eval_gpt4o_vision_proper.toml +29 -0
- examples/qwen_vl/configs/eval_gpt5nano_vision.toml +26 -0
- examples/qwen_vl/configs/eval_qwen3vl_vision.toml +26 -0
- examples/qwen_vl/configs/filter_qwen3vl_sft.toml +49 -0
- examples/qwen_vl/configs/filter_vision_sft.toml +52 -0
- examples/qwen_vl/configs/filter_vision_test.toml +8 -0
- examples/qwen_vl/configs/sft_qwen3_vl_2b_test.toml +54 -0
- examples/qwen_vl/crafter_gpt5nano_agent.py +308 -0
- examples/qwen_vl/crafter_qwen_vl_agent.py +300 -0
- examples/qwen_vl/run_vision_comparison.sh +61 -0
- examples/qwen_vl/run_vision_sft_pipeline.sh +175 -0
- examples/qwen_vl/test_image_validation.py +201 -0
- examples/qwen_vl/test_sft_vision_data.py +110 -0
- examples/rl/README.md +169 -0
- examples/rl/configs/eval_base_qwen.toml +17 -0
- examples/rl/configs/eval_rl_qwen.toml +13 -0
- examples/rl/configs/rl_from_base_qwen.toml +62 -0
- examples/rl/configs/rl_from_base_qwen17.toml +80 -0
- examples/rl/configs/rl_from_ft_qwen.toml +37 -0
- examples/rl/download_dataset.py +80 -0
- examples/rl/run_eval.py +436 -0
- examples/rl/run_rl_and_save.py +111 -0
- examples/rl/task_app/README.md +21 -0
- examples/rl/task_app/math_single_step.py +990 -0
- examples/rl/task_app/math_task_app.py +111 -0
- examples/run_crafter_demo.sh +10 -0
- examples/sdk_prompt_learning_example.py +55 -0
- examples/sft/README.md +139 -0
- examples/sft/configs/crafter_fft_qwen0p6b.toml +49 -0
- examples/sft/configs/crafter_lora_qwen0p6b.toml +49 -0
- examples/sft/evaluate.py +117 -0
- examples/sft/export_dataset.py +120 -0
- examples/sft/generate_traces.py +164 -0
- examples/swe/__init__.py +12 -0
- examples/swe/task_app/README.md +135 -0
- examples/swe/task_app/__init__.py +2 -0
- examples/swe/task_app/grpo_swe_mini.py +604 -0
- examples/swe/task_app/grpo_swe_mini_task_app.py +124 -0
- examples/swe/task_app/hosted/README.md +173 -0
- examples/swe/task_app/hosted/__init__.py +5 -0
- examples/swe/task_app/hosted/branching.py +143 -0
- examples/swe/task_app/hosted/environment_routes.py +1289 -0
- examples/swe/task_app/hosted/envs/__init__.py +1 -0
- examples/swe/task_app/hosted/envs/crafter/__init__.py +6 -0
- examples/swe/task_app/hosted/envs/crafter/app.py +1 -0
- examples/swe/task_app/hosted/envs/crafter/environment.py +522 -0
- examples/swe/task_app/hosted/envs/crafter/policy.py +478 -0
- examples/swe/task_app/hosted/envs/crafter/react_agent.py +108 -0
- examples/swe/task_app/hosted/envs/crafter/shared.py +305 -0
- examples/swe/task_app/hosted/envs/crafter/tools.py +47 -0
- examples/swe/task_app/hosted/envs/mini_swe/__init__.py +8 -0
- examples/swe/task_app/hosted/envs/mini_swe/environment.py +1191 -0
- examples/swe/task_app/hosted/envs/mini_swe/policy.py +355 -0
- examples/swe/task_app/hosted/envs/mini_swe/shared.py +83 -0
- examples/swe/task_app/hosted/envs/mini_swe/tools.py +96 -0
- examples/swe/task_app/hosted/hosted_app.py +204 -0
- examples/swe/task_app/hosted/inference/__init__.py +5 -0
- examples/swe/task_app/hosted/inference/openai_client.py +584 -0
- examples/swe/task_app/hosted/main.py +100 -0
- examples/swe/task_app/hosted/policy_routes.py +1094 -0
- examples/swe/task_app/hosted/registry.py +195 -0
- examples/swe/task_app/hosted/rollout.py +1905 -0
- examples/swe/task_app/hosted/storage/__init__.py +5 -0
- examples/swe/task_app/hosted/storage/volume.py +211 -0
- examples/swe/task_app/hosted/test_agents.py +161 -0
- examples/swe/task_app/hosted/test_service.py +136 -0
- examples/swe/task_app/hosted/utils.py +62 -0
- examples/swe/task_app/morph_backend.py +178 -0
- examples/task_apps/IMAGE_ONLY_EVAL_QUICKSTART.md +258 -0
- examples/task_apps/TESTING.md +275 -0
- examples/task_apps/banking77/__init__.py +6 -0
- examples/task_apps/banking77/banking77_task_app.py +912 -0
- examples/task_apps/banking77/deploy_wrapper.py +46 -0
- examples/task_apps/banking77_pipeline/__init__.py +6 -0
- examples/task_apps/banking77_pipeline/banking77_pipeline_task_app.py +489 -0
- examples/task_apps/banking77_pipeline/deploy_wrapper.py +50 -0
- examples/task_apps/crafter/CREATE_SFT_DATASET.md +286 -0
- examples/task_apps/crafter/EVAL_IMAGE_ONLY_RESULTS.md +152 -0
- examples/task_apps/crafter/FILTER_COMMAND_STATUS.md +187 -0
- examples/task_apps/crafter/FILTER_COMMAND_SUCCESS.md +281 -0
- examples/task_apps/crafter/QUERY_EXAMPLES.md +203 -0
- examples/task_apps/crafter/README_IMAGE_ONLY_EVAL.md +316 -0
- examples/task_apps/crafter/eval_image_only_gpt4o.toml +28 -0
- examples/task_apps/crafter/eval_text_only_groq_llama.toml +36 -0
- examples/task_apps/crafter/filter_sft_dataset.toml +16 -0
- examples/task_apps/crafter/task_app/README.md +42 -0
- examples/task_apps/crafter/task_app/__init__.py +5 -0
- examples/task_apps/crafter/task_app/grpo_crafter.py +1055 -0
- examples/task_apps/crafter/task_app/grpo_crafter_task_app.py +146 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/README.md +173 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/__init__.py +5 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/branching.py +143 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/environment_routes.py +1226 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/__init__.py +1 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/__init__.py +6 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/app.py +1 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/environment.py +532 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/policy.py +583 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/react_agent.py +122 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/shared.py +305 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/tools.py +47 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/hosted_app.py +253 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/inference/__init__.py +5 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/inference/openai_client.py +999 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/main.py +100 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/policy_routes.py +1252 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/registry.py +195 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/rollout.py +2233 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/storage/__init__.py +5 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/storage/volume.py +211 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/test_agents.py +161 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/test_service.py +136 -0
- examples/task_apps/crafter/task_app/synth_envs_hosted/utils.py +411 -0
- examples/task_apps/dev/pokemon_emerald/__init__.py +2 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/README.md +811 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/__init__.py +120 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/action.py +160 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/memory.py +155 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/perception.py +69 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/planning.py +96 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/simple.py +1502 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/agent/system_prompt.py +4 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/grab_map.py +68 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/manual.py +216 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/__init__.py +35 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/emerald_utils.py +631 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/emulator.py +1544 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/enums.py +1428 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/memory_reader.py +4848 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/types.py +41 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pokemon_env/utils.py +298 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/pyproject.toml +95 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/run.py +204 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/server/app.py +2152 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/server/client.py +429 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/server/frame_server.py +155 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/README.md +78 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/run_tests.py +122 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_agent_direct.py +76 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_agent_prompts.py +413 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_battle_state_formatting.py +204 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_dialogue_detection.py +133 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_dialogue_detection_comprehensive.py +229 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_direct_agent_emulator.py +300 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_fps_adjustment_pytest.py +205 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_house_to_outside_direct.py +200 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_house_to_outside_transition.py +284 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_map_ground_truth_comparison.py +468 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_memory_map.py +575 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_server_map_validation.py +311 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_torchic_state.py +259 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/anticheat.py +372 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/checkpoint.py +296 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/error_handler.py +275 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/get_local_ip.py +22 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/helpers.py +44 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/llm_logger.py +514 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/map_formatter.py +415 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/map_stitcher.py +1763 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/map_stitcher_singleton.py +33 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/map_trimmer.py +106 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/map_visualizer.py +334 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/ocr_dialogue.py +1020 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/recording.py +188 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/state_formatter.py +1481 -0
- examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils/vlm.py +862 -0
- examples/task_apps/dev/pokemon_emerald/modal_app.py +114 -0
- examples/task_apps/dev/pokemon_emerald/task_app/README.md +81 -0
- examples/task_apps/dev/pokemon_emerald/task_app/__init__.py +6 -0
- examples/task_apps/dev/pokemon_emerald/task_app/pokemon_emerald.py +685 -0
- examples/task_apps/enron/__init__.py +2 -0
- examples/task_apps/enron/eval_groq_qwen32.toml +16 -0
- examples/task_apps/enron/filter_sft.toml +5 -0
- examples/task_apps/enron/task_app/README.md +14 -0
- examples/task_apps/enron/task_app/__init__.py +1 -0
- examples/task_apps/enron/task_app/grpo_enron.py +906 -0
- examples/task_apps/enron/task_app/grpo_enron_task_app.py +146 -0
- examples/task_apps/enron/tests/__init__.py +4 -0
- examples/task_apps/enron/tests/conftest.py +115 -0
- examples/task_apps/enron/tests/integration/__init__.py +4 -0
- examples/task_apps/enron/tests/integration/test_enron_eval.py +179 -0
- examples/task_apps/enron/tests/integration/test_enron_rollout.py +135 -0
- examples/task_apps/enron/tests/unit/__init__.py +4 -0
- examples/task_apps/enron/tests/unit/test_enron_environment.py +126 -0
- examples/task_apps/gepa_benchmarks/__init__.py +7 -0
- examples/task_apps/gepa_benchmarks/common.py +260 -0
- examples/task_apps/gepa_benchmarks/hotpotqa_task_app.py +507 -0
- examples/task_apps/gepa_benchmarks/hover_task_app.py +436 -0
- examples/task_apps/gepa_benchmarks/ifbench_task_app.py +563 -0
- examples/task_apps/gepa_benchmarks/pupa_task_app.py +460 -0
- examples/task_apps/math/README.md +21 -0
- examples/task_apps/math/math_single_step.py +1000 -0
- examples/task_apps/math/math_task_app.py +115 -0
- examples/task_apps/pokemon_battle/__init__.py +2 -0
- examples/task_apps/pokemon_battle/modal_app.py +104 -0
- examples/task_apps/pokemon_battle/task_app/README.md +68 -0
- examples/task_apps/pokemon_battle/task_app/__init__.py +6 -0
- examples/task_apps/pokemon_battle/task_app/pokemon_showdown.py +932 -0
- examples/task_apps/pokemon_red/EVAL_IMAGE_ONLY_COMPLETE.md +283 -0
- examples/task_apps/pokemon_red/EVAL_IMAGE_ONLY_STATUS.md +155 -0
- examples/task_apps/pokemon_red/README.md +356 -0
- examples/task_apps/pokemon_red/README_IMAGE_ONLY_EVAL.md +428 -0
- examples/task_apps/pokemon_red/__init__.py +3 -0
- examples/task_apps/pokemon_red/eval_image_only_gpt4o.toml +30 -0
- examples/task_apps/pokemon_red/eval_pokemon_red_policy.py +224 -0
- examples/task_apps/pokemon_red/pallet_town_rl_config.toml +75 -0
- examples/task_apps/pokemon_red/task_app.py +1048 -0
- examples/task_apps/pokemon_red/test_pallet_town_rewards.py +193 -0
- examples/task_apps/sokoban/README.md +306 -0
- examples/task_apps/sokoban/__init__.py +3 -0
- examples/task_apps/sokoban/eval_groq_qwen32.toml +16 -0
- examples/task_apps/sokoban/eval_openai_gpt5.toml +16 -0
- examples/task_apps/sokoban/filter_sft.toml +5 -0
- examples/task_apps/sokoban/task_app.py +1058 -0
- examples/task_apps/sokoban/tests/__init__.py +4 -0
- examples/task_apps/sokoban/tests/conftest.py +113 -0
- examples/task_apps/sokoban/tests/integration/__init__.py +4 -0
- examples/task_apps/sokoban/tests/integration/test_sokoban_eval.py +57 -0
- examples/task_apps/sokoban/tests/integration/test_sokoban_rollout.py +198 -0
- examples/task_apps/sokoban/tests/unit/__init__.py +4 -0
- examples/task_apps/sokoban/tests/unit/test_sokoban_environment.py +114 -0
- examples/task_apps/verilog/__init__.py +1 -0
- examples/task_apps/verilog/eval_groq_qwen32b.toml +22 -0
- examples/task_apps/verilog/filter_sft.toml +5 -0
- examples/task_apps/verilog/task_app/README.md +12 -0
- examples/task_apps/verilog/task_app/__init__.py +1 -0
- examples/task_apps/verilog/task_app/grpo_verilog.py +1166 -0
- examples/task_apps/verilog/task_app/grpo_verilog_task_app.py +145 -0
- examples/task_apps/verilog/tests/__init__.py +4 -0
- examples/task_apps/verilog/tests/conftest.py +115 -0
- examples/task_apps/verilog/tests/integration/__init__.py +4 -0
- examples/task_apps/verilog/tests/integration/test_verilog_eval.py +181 -0
- examples/task_apps/verilog/tests/integration/test_verilog_rollout.py +55 -0
- examples/task_apps/verilog/tests/unit/__init__.py +4 -0
- examples/task_apps/verilog/tests/unit/test_verilog_scoring.py +118 -0
- examples/tunnel_gepa_banking77/README.md +106 -0
- examples/tunnel_gepa_banking77/banking77_gepa_tunnel.toml +95 -0
- examples/tunnel_gepa_banking77/keep_tunnel_running.py +60 -0
- examples/tunnel_gepa_banking77/run_gepa_with_tunnel.sh +226 -0
- examples/vlm/PROPOSAL.md +53 -0
- examples/vlm/README.md +68 -0
- examples/vlm/configs/crafter_vlm_gpt4o.toml +49 -0
- examples/vlm/crafter_image_only_agent.py +207 -0
- examples/vlm/crafter_openai_vlm_agent.py +275 -0
- examples/vlm/filter_image_rows.py +63 -0
- examples/vlm/run_crafter_vlm_benchmark.py +316 -0
- examples/warming_up_to_rl/_utils.py +92 -0
- examples/warming_up_to_rl/analyze_trace_db.py +422 -0
- examples/warming_up_to_rl/configs/crafter_fft.toml +53 -0
- examples/warming_up_to_rl/configs/crafter_fft_4b.toml +54 -0
- examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml +22 -0
- examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml +15 -0
- examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml +24 -0
- examples/warming_up_to_rl/configs/eval_stepwise_complex.toml +35 -0
- examples/warming_up_to_rl/configs/eval_stepwise_consistent.toml +26 -0
- examples/warming_up_to_rl/configs/eval_stepwise_per_achievement.toml +36 -0
- examples/warming_up_to_rl/configs/eval_stepwise_simple.toml +32 -0
- examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml +85 -0
- examples/warming_up_to_rl/configs/rl_from_ft.toml +58 -0
- examples/warming_up_to_rl/export_trace_sft.py +837 -0
- examples/warming_up_to_rl/groq_test.py +97 -0
- examples/warming_up_to_rl/manage_secrets.py +131 -0
- examples/warming_up_to_rl/old/event_rewards.md +234 -0
- examples/warming_up_to_rl/old/notes.md +73 -0
- examples/warming_up_to_rl/readme.md +110 -0
- examples/warming_up_to_rl/run_eval.py +736 -0
- examples/warming_up_to_rl/run_fft_and_save.py +380 -0
- examples/warming_up_to_rl/run_local_rollout.py +239 -0
- examples/warming_up_to_rl/run_local_rollout_modal.py +248 -0
- examples/warming_up_to_rl/run_local_rollout_parallel.py +405 -0
- examples/warming_up_to_rl/run_local_rollout_traced.py +477 -0
- examples/warming_up_to_rl/run_rl_and_save.py +124 -0
- examples/warming_up_to_rl/run_rollout_remote.py +156 -0
- examples/warming_up_to_rl/task_app/README.md +42 -0
- examples/warming_up_to_rl/task_app/grpo_crafter.py +876 -0
- examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py +135 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/README.md +173 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/__init__.py +5 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/branching.py +143 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/environment_routes.py +1226 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/__init__.py +1 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/__init__.py +6 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/app.py +1 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py +522 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py +454 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/react_agent.py +108 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/shared.py +305 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/tools.py +47 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py +253 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/__init__.py +5 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py +729 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/main.py +100 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py +1114 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/registry.py +195 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py +1891 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/__init__.py +5 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/volume.py +211 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_agents.py +161 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_service.py +137 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/utils.py +129 -0
- examples/workflows/math_rl/configs/eval_base_qwen.toml +15 -0
- examples/workflows/math_rl/configs/eval_rl_qwen.toml +11 -0
- examples/workflows/math_rl/configs/rl_from_base_qwen.toml +62 -0
- examples/workflows/math_rl/configs/rl_from_base_qwen17.toml +80 -0
- examples/workflows/math_rl/configs/rl_from_ft_qwen.toml +35 -0
- examples/workflows/math_rl/download_dataset.py +80 -0
- examples/workflows/math_rl/run_eval.py +436 -0
- examples/workflows/math_rl/run_rl_and_save.py +111 -0
- synth_ai/__init__.py +47 -23
- synth_ai/_utils/__init__.py +47 -0
- synth_ai/_utils/base_url.py +10 -0
- synth_ai/_utils/http.py +10 -0
- synth_ai/_utils/prompts.py +10 -0
- synth_ai/_utils/task_app_state.py +12 -0
- synth_ai/_utils/user_config.py +10 -0
- synth_ai/api/models/supported.py +514 -0
- synth_ai/api/train/__init__.py +63 -0
- synth_ai/api/train/builders.py +473 -0
- synth_ai/api/train/cli.py +1185 -0
- synth_ai/api/train/config_finder.py +246 -0
- synth_ai/api/train/configs/__init__.py +65 -0
- synth_ai/api/train/configs/prompt_learning.py +496 -0
- synth_ai/api/train/configs/rl.py +188 -0
- synth_ai/api/train/configs/sft.py +99 -0
- synth_ai/api/train/configs/shared.py +81 -0
- synth_ai/api/train/env_resolver.py +352 -0
- synth_ai/api/train/pollers.py +91 -0
- synth_ai/api/train/prompt_learning.py +425 -0
- synth_ai/api/train/sft.py +390 -0
- synth_ai/api/train/supported_algos.py +147 -0
- synth_ai/api/train/task_app.py +195 -0
- synth_ai/api/train/utils.py +244 -0
- synth_ai/api/train/validators.py +1117 -0
- synth_ai/api/tunnel.py +49 -0
- synth_ai/auth/credentials.py +94 -0
- synth_ai/baseline/__init__.py +25 -0
- synth_ai/baseline/config.py +209 -0
- synth_ai/baseline/discovery.py +214 -0
- synth_ai/baseline/execution.py +146 -0
- synth_ai/cfgs.py +227 -0
- synth_ai/cli/__init__.py +90 -45
- synth_ai/cli/_modal_wrapper.py +31 -0
- synth_ai/cli/_storage.py +20 -0
- synth_ai/cli/_typer_patch.py +47 -0
- synth_ai/cli/_validate_task_app.py +29 -0
- synth_ai/cli/balance.py +16 -4
- synth_ai/cli/calc.py +36 -21
- synth_ai/cli/claude.py +70 -0
- synth_ai/cli/codex.py +267 -0
- synth_ai/cli/commands/__init__.py +18 -0
- synth_ai/cli/commands/baseline/__init__.py +12 -0
- synth_ai/cli/commands/baseline/core.py +637 -0
- synth_ai/cli/commands/baseline/list.py +93 -0
- synth_ai/cli/commands/demo/__init__.py +6 -0
- synth_ai/cli/commands/demo/core.py +163 -0
- synth_ai/cli/commands/eval/__init__.py +19 -0
- synth_ai/cli/commands/eval/core.py +1112 -0
- synth_ai/cli/commands/eval/errors.py +81 -0
- synth_ai/cli/commands/eval/validation.py +133 -0
- synth_ai/cli/commands/filter/__init__.py +12 -0
- synth_ai/cli/commands/filter/core.py +424 -0
- synth_ai/cli/commands/filter/errors.py +55 -0
- synth_ai/cli/commands/filter/validation.py +77 -0
- synth_ai/cli/commands/help/__init__.py +185 -0
- synth_ai/cli/commands/help/core.py +72 -0
- synth_ai/cli/commands/smoke/__init__.py +7 -0
- synth_ai/cli/commands/smoke/core.py +1437 -0
- synth_ai/cli/commands/status/__init__.py +66 -0
- synth_ai/cli/commands/status/client.py +192 -0
- synth_ai/cli/commands/status/config.py +92 -0
- synth_ai/cli/commands/status/errors.py +20 -0
- synth_ai/cli/commands/status/formatters.py +164 -0
- synth_ai/cli/commands/status/subcommands/__init__.py +9 -0
- synth_ai/cli/commands/status/subcommands/files.py +79 -0
- synth_ai/cli/commands/status/subcommands/jobs.py +334 -0
- synth_ai/cli/commands/status/subcommands/models.py +79 -0
- synth_ai/cli/commands/status/subcommands/pricing.py +22 -0
- synth_ai/cli/commands/status/subcommands/runs.py +81 -0
- synth_ai/cli/commands/status/subcommands/session.py +183 -0
- synth_ai/cli/commands/status/subcommands/summary.py +47 -0
- synth_ai/cli/commands/status/subcommands/usage.py +203 -0
- synth_ai/cli/commands/status/utils.py +114 -0
- synth_ai/cli/commands/train/__init__.py +53 -0
- synth_ai/cli/commands/train/core.py +21 -0
- synth_ai/cli/commands/train/errors.py +117 -0
- synth_ai/cli/commands/train/judge_schemas.py +200 -0
- synth_ai/cli/commands/train/judge_validation.py +305 -0
- synth_ai/cli/commands/train/validation.py +386 -0
- synth_ai/cli/demo.py +32 -140
- synth_ai/cli/deploy.py +233 -0
- synth_ai/cli/eval/__init__.py +36 -0
- synth_ai/cli/eval/core.py +5 -0
- synth_ai/cli/eval/errors.py +31 -0
- synth_ai/cli/eval/validation.py +5 -0
- synth_ai/cli/filter/__init__.py +28 -0
- synth_ai/cli/filter/core.py +5 -0
- synth_ai/cli/filter/errors.py +23 -0
- synth_ai/cli/filter/validation.py +5 -0
- synth_ai/cli/legacy_root_backup.py +28 -22
- synth_ai/cli/lib/__init__.py +10 -0
- synth_ai/cli/lib/task_app_discovery.py +7 -0
- synth_ai/cli/lib/task_app_env.py +518 -0
- synth_ai/cli/mcp.py +34 -0
- synth_ai/cli/modal_serve/__init__.py +12 -0
- synth_ai/cli/modal_serve/core.py +14 -0
- synth_ai/cli/modal_serve/errors.py +8 -0
- synth_ai/cli/modal_serve/validation.py +11 -0
- synth_ai/cli/opencode.py +256 -0
- synth_ai/cli/recent.py +13 -7
- synth_ai/cli/rl_demo.py +166 -114
- synth_ai/cli/root.py +143 -112
- synth_ai/cli/serve/__init__.py +12 -0
- synth_ai/cli/serve/core.py +14 -0
- synth_ai/cli/serve/errors.py +8 -0
- synth_ai/cli/serve/validation.py +11 -0
- synth_ai/cli/setup.py +49 -0
- synth_ai/cli/status.py +7 -125
- synth_ai/cli/task_app_deploy.py +7 -0
- synth_ai/cli/task_app_list.py +25 -0
- synth_ai/cli/task_app_modal_serve.py +11 -0
- synth_ai/cli/task_app_serve.py +11 -0
- synth_ai/cli/task_apps.py +3134 -0
- synth_ai/cli/traces.py +9 -5
- synth_ai/cli/train/__init__.py +12 -0
- synth_ai/cli/train/core.py +21 -0
- synth_ai/cli/train/errors.py +8 -0
- synth_ai/cli/train/validation.py +24 -0
- synth_ai/cli/train.py +5 -0
- synth_ai/cli/turso.py +73 -0
- synth_ai/cli/watch.py +13 -18
- synth_ai/demos/__init__.py +10 -0
- synth_ai/demos/core/__init__.py +28 -1
- synth_ai/demos/core/cli.py +745 -416
- synth_ai/demos/crafter/__init__.py +1 -0
- synth_ai/demos/crafter/crafter_fft_4b.toml +55 -0
- synth_ai/demos/crafter/grpo_crafter_task_app.py +185 -0
- synth_ai/demos/crafter/rl_from_base_qwen4b.toml +74 -0
- synth_ai/demos/demo_registry.py +176 -0
- synth_ai/demos/demo_task_apps/__init__.py +7 -1
- synth_ai/demos/demo_task_apps/core.py +75 -37
- synth_ai/demos/demo_task_apps/crafter/__init__.py +1 -0
- synth_ai/demos/demo_task_apps/crafter/configs/crafter_fft_4b.toml +53 -0
- synth_ai/demos/demo_task_apps/crafter/configs/rl_from_base_qwen4b.toml +73 -0
- synth_ai/demos/demo_task_apps/crafter/grpo_crafter_task_app.py +184 -0
- synth_ai/demos/demo_task_apps/math/_common.py +1 -2
- synth_ai/demos/demo_task_apps/math/app.py +2 -1
- synth_ai/demos/demo_task_apps/math/config.toml +55 -110
- synth_ai/demos/demo_task_apps/math/deploy_modal.py +3 -6
- synth_ai/demos/demo_task_apps/math/modal_task_app.py +491 -166
- synth_ai/demos/demo_task_apps/math/task_app_entry.py +37 -0
- synth_ai/demos/math/__init__.py +1 -0
- synth_ai/demos/math/_common.py +16 -0
- synth_ai/demos/math/app.py +38 -0
- synth_ai/demos/math/config.toml +76 -0
- synth_ai/demos/math/deploy_modal.py +54 -0
- synth_ai/demos/math/modal_task_app.py +703 -0
- synth_ai/demos/math/task_app_entry.py +51 -0
- synth_ai/environments/environment/core.py +7 -1
- synth_ai/environments/examples/bandit/engine.py +12 -5
- synth_ai/environments/examples/bandit/environment.py +0 -1
- synth_ai/environments/examples/bandit/taskset.py +4 -4
- synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py +7 -4
- synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py +9 -5
- synth_ai/environments/examples/crafter_classic/environment.py +93 -2
- synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py +4 -3
- synth_ai/environments/examples/enron/engine.py +7 -2
- synth_ai/environments/examples/enron/environment.py +68 -0
- synth_ai/environments/examples/red/engine.py +60 -12
- synth_ai/environments/examples/red/engine_helpers/memory_map.py +7 -0
- synth_ai/environments/examples/red/engine_helpers/reward_components.py +151 -179
- synth_ai/environments/examples/red/engine_helpers/reward_library/pallet_town_progression.py +477 -0
- synth_ai/environments/examples/red/engine_helpers/state_extraction.py +32 -0
- synth_ai/environments/examples/red/environment.py +86 -0
- synth_ai/environments/examples/red/trace_hooks_v3.py +168 -0
- synth_ai/environments/examples/sokoban/taskset.py +116 -0
- synth_ai/environments/examples/verilog/engine.py +104 -12
- synth_ai/environments/examples/wordle/environment.py +0 -1
- synth_ai/environments/reproducibility/tree.py +5 -6
- synth_ai/environments/service/app.py +11 -12
- synth_ai/environments/service/core_routes.py +10 -9
- synth_ai/environments/stateful/engine.py +1 -1
- synth_ai/environments/tasks/core.py +1 -0
- synth_ai/environments/tasks/filters.py +5 -6
- synth_ai/environments/tasks/utils.py +4 -5
- synth_ai/evals/__init__.py +15 -0
- synth_ai/evals/base.py +14 -5
- synth_ai/evals/client.py +82 -0
- synth_ai/evals/types.py +42 -0
- synth_ai/http.py +8 -22
- synth_ai/http_client.py +45 -12
- synth_ai/inference/__init__.py +0 -2
- synth_ai/inference/client.py +21 -7
- synth_ai/jobs/client.py +129 -80
- synth_ai/judge_schemas.py +127 -0
- synth_ai/learning/__init__.py +51 -6
- synth_ai/learning/algorithms.py +14 -0
- synth_ai/learning/client.py +122 -30
- synth_ai/learning/config.py +2 -40
- synth_ai/learning/constants.py +0 -2
- synth_ai/learning/ft_client.py +4 -56
- synth_ai/learning/health.py +14 -8
- synth_ai/learning/jobs.py +43 -47
- synth_ai/learning/prompt_learning_client.py +276 -0
- synth_ai/learning/prompt_learning_types.py +185 -0
- synth_ai/{rl → learning/rl}/__init__.py +14 -5
- synth_ai/learning/rl/client.py +269 -0
- synth_ai/learning/rl/config.py +31 -0
- synth_ai/{rl → learning/rl}/contracts.py +5 -10
- synth_ai/{rl → learning/rl}/env_keys.py +45 -16
- synth_ai/learning/rl/secrets.py +13 -0
- synth_ai/learning/rl_client.py +2 -253
- synth_ai/learning/sft/__init__.py +29 -0
- synth_ai/learning/sft/client.py +68 -0
- synth_ai/learning/sft/config.py +270 -0
- synth_ai/learning/sft/data.py +698 -0
- synth_ai/learning/sse.py +25 -26
- synth_ai/learning/validators.py +29 -25
- synth_ai/mcp/__init__.py +5 -0
- synth_ai/mcp/__main__.py +8 -0
- synth_ai/mcp/main.py +254 -0
- synth_ai/mcp/setup.py +100 -0
- synth_ai/modal.py +257 -0
- synth_ai/pricing/__init__.py +3 -0
- synth_ai/pricing/model_pricing.py +64 -0
- synth_ai/session/__init__.py +75 -0
- synth_ai/session/client.py +383 -0
- synth_ai/session/constants.py +63 -0
- synth_ai/session/exceptions.py +105 -0
- synth_ai/session/manager.py +139 -0
- synth_ai/session/models.py +89 -0
- synth_ai/session/query.py +110 -0
- synth_ai/spec/__init__.py +46 -0
- synth_ai/spec/dataclasses.py +149 -0
- synth_ai/spec/loader.py +144 -0
- synth_ai/spec/serializer.py +199 -0
- synth_ai/spec/validation.py +250 -0
- synth_ai/streaming/__init__.py +29 -0
- synth_ai/streaming/config.py +94 -0
- synth_ai/streaming/handlers.py +589 -0
- synth_ai/streaming/streamer.py +320 -0
- synth_ai/streaming/types.py +95 -0
- synth_ai/task/__init__.py +116 -3
- synth_ai/task/apps/__init__.py +132 -0
- synth_ai/task/auth.py +165 -0
- synth_ai/task/client.py +167 -0
- synth_ai/task/config.py +261 -0
- synth_ai/task/contracts.py +173 -57
- synth_ai/task/datasets.py +108 -0
- synth_ai/task/errors.py +50 -0
- synth_ai/task/health.py +17 -11
- synth_ai/task/inference_api.py +101 -0
- synth_ai/task/json.py +111 -0
- synth_ai/task/proxy.py +251 -0
- synth_ai/task/rubrics/__init__.py +55 -0
- synth_ai/task/rubrics/loaders.py +156 -0
- synth_ai/task/rubrics/models.py +57 -0
- synth_ai/task/rubrics/scoring.py +116 -0
- synth_ai/task/rubrics/strict.py +149 -0
- synth_ai/task/rubrics.py +219 -0
- synth_ai/task/server.py +432 -0
- synth_ai/task/trace_correlation_helpers.py +328 -0
- synth_ai/task/tracing_utils.py +95 -0
- synth_ai/task/validators.py +449 -6
- synth_ai/task/vendors.py +59 -0
- synth_ai/tracing_v3/__init__.py +4 -0
- synth_ai/tracing_v3/abstractions.py +21 -4
- synth_ai/tracing_v3/config.py +167 -22
- synth_ai/tracing_v3/constants.py +21 -0
- synth_ai/tracing_v3/db_config.py +42 -29
- synth_ai/tracing_v3/decorators.py +80 -45
- synth_ai/tracing_v3/examples/basic_usage.py +15 -9
- synth_ai/tracing_v3/hooks.py +6 -4
- synth_ai/tracing_v3/llm_call_record_helpers.py +161 -61
- synth_ai/tracing_v3/migration_helper.py +1 -2
- synth_ai/tracing_v3/replica_sync.py +12 -7
- synth_ai/tracing_v3/serialization.py +130 -0
- synth_ai/tracing_v3/session_tracer.py +86 -21
- synth_ai/tracing_v3/storage/base.py +98 -12
- synth_ai/tracing_v3/storage/config.py +63 -16
- synth_ai/tracing_v3/storage/factory.py +11 -9
- synth_ai/tracing_v3/storage/utils.py +15 -11
- synth_ai/tracing_v3/trace_utils.py +317 -0
- synth_ai/tracing_v3/turso/__init__.py +8 -21
- synth_ai/tracing_v3/turso/daemon.py +123 -15
- synth_ai/tracing_v3/turso/models.py +5 -2
- synth_ai/tracing_v3/turso/native_manager.py +1293 -0
- synth_ai/tracing_v3/utils.py +5 -4
- synth_ai/tunnel.py +143 -0
- synth_ai/tunnel_deploy.py +278 -0
- synth_ai/types.py +8 -0
- synth_ai/urls.py +11 -0
- synth_ai/utils/__init__.py +166 -0
- synth_ai/utils/agents.py +74 -0
- synth_ai/utils/apps.py +152 -0
- synth_ai/utils/base_url.py +94 -0
- synth_ai/utils/bin.py +39 -0
- synth_ai/utils/claude.py +36 -0
- synth_ai/utils/cli.py +284 -0
- synth_ai/utils/config.py +81 -0
- synth_ai/utils/env.py +346 -0
- synth_ai/utils/errors.py +85 -0
- synth_ai/utils/http.py +172 -0
- synth_ai/utils/json.py +72 -0
- synth_ai/utils/log_filter.py +99 -0
- synth_ai/utils/logging.py +198 -0
- synth_ai/utils/modal.py +299 -0
- synth_ai/utils/paths.py +95 -0
- synth_ai/utils/process.py +233 -0
- synth_ai/utils/prompts.py +39 -0
- synth_ai/utils/sqld.py +122 -0
- synth_ai/utils/ssl.py +25 -0
- synth_ai/utils/task_app_discovery.py +882 -0
- synth_ai/utils/task_app_env.py +186 -0
- synth_ai/utils/task_app_state.py +318 -0
- synth_ai/utils/tunnel/__init__.py +12 -0
- synth_ai/utils/tunnel/config.py +55 -0
- synth_ai/utils/user_config.py +137 -0
- synth_ai/uvicorn.py +77 -0
- synth_ai-0.2.23.dev3.dist-info/METADATA +357 -0
- synth_ai-0.2.23.dev3.dist-info/RECORD +983 -0
- {synth_ai-0.2.8.dev4.dist-info → synth_ai-0.2.23.dev3.dist-info}/entry_points.txt +0 -1
- {synth_ai-0.2.8.dev4.dist-info → synth_ai-0.2.23.dev3.dist-info}/top_level.txt +1 -0
- synth_ai/cli/man.py +0 -106
- synth_ai/core/experiment.py +0 -15
- synth_ai/core/system.py +0 -15
- synth_ai/environments/examples/sokoban/units/astar_common.py +0 -95
- synth_ai/experimental/synth_oss.py +0 -446
- synth_ai/handshake.py +0 -63
- synth_ai/install_sqld.sh +0 -40
- synth_ai/learning/offline/dpo.py +0 -0
- synth_ai/learning/offline/providers.py +0 -7
- synth_ai/learning/offline/sft.py +0 -0
- synth_ai/learning/offline/shared.py +0 -0
- synth_ai/learning/online/grpo.py +0 -0
- synth_ai/learning/online/irft.py +0 -0
- synth_ai/learning/prompts/banking77_injection_eval.py +0 -168
- synth_ai/learning/prompts/gepa.py +0 -0
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +0 -213
- synth_ai/learning/prompts/mipro.py +0 -289
- synth_ai/learning/prompts/random_search.py +0 -246
- synth_ai/learning/prompts/run_mipro_banking77.py +0 -172
- synth_ai/learning/prompts/run_random_search_banking77.py +0 -324
- synth_ai/lm/__init__.py +0 -51
- synth_ai/lm/caching/constants.py +0 -6
- synth_ai/lm/caching/dbs.py +0 -0
- synth_ai/lm/caching/ephemeral.py +0 -102
- synth_ai/lm/caching/handler.py +0 -137
- synth_ai/lm/caching/initialize.py +0 -11
- synth_ai/lm/caching/persistent.py +0 -114
- synth_ai/lm/config.py +0 -110
- synth_ai/lm/constants.py +0 -32
- synth_ai/lm/core/__init__.py +0 -8
- synth_ai/lm/core/all.py +0 -73
- synth_ai/lm/core/exceptions.py +0 -7
- synth_ai/lm/core/main.py +0 -319
- synth_ai/lm/core/main_v3.py +0 -594
- synth_ai/lm/core/synth_models.py +0 -48
- synth_ai/lm/core/vendor_clients.py +0 -188
- synth_ai/lm/cost/monitor.py +0 -1
- synth_ai/lm/cost/statefulness.py +0 -1
- synth_ai/lm/injection.py +0 -80
- synth_ai/lm/overrides.py +0 -206
- synth_ai/lm/provider_support/__init__.py +0 -8
- synth_ai/lm/provider_support/anthropic.py +0 -972
- synth_ai/lm/provider_support/openai.py +0 -1139
- synth_ai/lm/provider_support/suppress_logging.py +0 -31
- synth_ai/lm/structured_outputs/handler.py +0 -440
- synth_ai/lm/structured_outputs/inject.py +0 -297
- synth_ai/lm/structured_outputs/rehabilitate.py +0 -185
- synth_ai/lm/tools/__init__.py +0 -3
- synth_ai/lm/tools/base.py +0 -172
- synth_ai/lm/unified_interface.py +0 -202
- synth_ai/lm/vendors/base.py +0 -81
- synth_ai/lm/vendors/core/anthropic_api.py +0 -387
- synth_ai/lm/vendors/core/gemini_api.py +0 -292
- synth_ai/lm/vendors/core/mistral_api.py +0 -322
- synth_ai/lm/vendors/core/openai_api.py +0 -225
- synth_ai/lm/vendors/core/synth_dev_api.py +0 -0
- synth_ai/lm/vendors/local/ollama.py +0 -0
- synth_ai/lm/vendors/openai_standard.py +0 -780
- synth_ai/lm/vendors/openai_standard_responses.py +0 -256
- synth_ai/lm/vendors/retries.py +0 -22
- synth_ai/lm/vendors/supported/custom_endpoint.py +0 -417
- synth_ai/lm/vendors/supported/deepseek.py +0 -69
- synth_ai/lm/vendors/supported/grok.py +0 -75
- synth_ai/lm/vendors/supported/groq.py +0 -16
- synth_ai/lm/vendors/supported/ollama.py +0 -15
- synth_ai/lm/vendors/supported/openrouter.py +0 -74
- synth_ai/lm/vendors/supported/together.py +0 -11
- synth_ai/lm/vendors/synth_client.py +0 -808
- synth_ai/lm/warmup.py +0 -186
- synth_ai/rl/secrets.py +0 -19
- synth_ai/scripts/verify_rewards.py +0 -100
- synth_ai/tracing/__init__.py +0 -30
- synth_ai/tracing_v1/__init__.py +0 -33
- synth_ai/tracing_v3/turso/manager.py +0 -760
- synth_ai/v0/tracing/abstractions.py +0 -224
- synth_ai/v0/tracing/base_client.py +0 -91
- synth_ai/v0/tracing/client_manager.py +0 -131
- synth_ai/v0/tracing/config.py +0 -142
- synth_ai/v0/tracing/context.py +0 -146
- synth_ai/v0/tracing/decorators.py +0 -682
- synth_ai/v0/tracing/events/__init__.py +0 -0
- synth_ai/v0/tracing/events/manage.py +0 -147
- synth_ai/v0/tracing/events/scope.py +0 -86
- synth_ai/v0/tracing/events/store.py +0 -228
- synth_ai/v0/tracing/immediate_client.py +0 -151
- synth_ai/v0/tracing/local.py +0 -18
- synth_ai/v0/tracing/log_client_base.py +0 -73
- synth_ai/v0/tracing/retry_queue.py +0 -186
- synth_ai/v0/tracing/trackers.py +0 -515
- synth_ai/v0/tracing/upload.py +0 -512
- synth_ai/v0/tracing/utils.py +0 -9
- synth_ai/v0/tracing_v1/__init__.py +0 -16
- synth_ai/v0/tracing_v1/abstractions.py +0 -224
- synth_ai/v0/tracing_v1/base_client.py +0 -91
- synth_ai/v0/tracing_v1/client_manager.py +0 -131
- synth_ai/v0/tracing_v1/config.py +0 -142
- synth_ai/v0/tracing_v1/context.py +0 -146
- synth_ai/v0/tracing_v1/decorators.py +0 -703
- synth_ai/v0/tracing_v1/events/__init__.py +0 -0
- synth_ai/v0/tracing_v1/events/manage.py +0 -147
- synth_ai/v0/tracing_v1/events/scope.py +0 -86
- synth_ai/v0/tracing_v1/events/store.py +0 -228
- synth_ai/v0/tracing_v1/immediate_client.py +0 -151
- synth_ai/v0/tracing_v1/local.py +0 -18
- synth_ai/v0/tracing_v1/log_client_base.py +0 -73
- synth_ai/v0/tracing_v1/retry_queue.py +0 -186
- synth_ai/v0/tracing_v1/trackers.py +0 -515
- synth_ai/v0/tracing_v1/upload.py +0 -527
- synth_ai/v0/tracing_v1/utils.py +0 -9
- synth_ai/zyk/__init__.py +0 -30
- synth_ai-0.2.8.dev4.dist-info/METADATA +0 -129
- synth_ai-0.2.8.dev4.dist-info/RECORD +0 -420
- {synth_ai/lm/caching → examples/task_apps}/__init__.py +0 -0
- {synth_ai/lm/cost → examples/task_apps/crafter}/__init__.py +0 -0
- {synth_ai/lm/structured_outputs → examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/server}/__init__.py +0 -0
- {synth_ai/lm/vendors → examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests}/__init__.py +0 -0
- {synth_ai/lm/vendors/core → examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/utils}/__init__.py +0 -0
- {synth_ai/lm/vendors/local → examples/task_apps/math}/__init__.py +0 -0
- {synth_ai/lm/vendors/supported → examples/workflows}/__init__.py +0 -0
- {synth_ai/v0/tracing → examples/workflows/math_rl}/__init__.py +0 -0
- /synth_ai/{compound/cais.py → cli/__main__.py} +0 -0
- /synth_ai/{learning/filtering.py → py.typed} +0 -0
- {synth_ai-0.2.8.dev4.dist-info → synth_ai-0.2.23.dev3.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.8.dev4.dist-info → synth_ai-0.2.23.dev3.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Deploy Banking77 task app locally for GEPA optimization
|
|
3
|
+
|
|
4
|
+
set -e
|
|
5
|
+
|
|
6
|
+
echo "🚀 Deploying Banking77 Task App..."
|
|
7
|
+
echo "=================================="
|
|
8
|
+
|
|
9
|
+
# Load .env file if it exists (for OneShot pair programming sessions)
|
|
10
|
+
# Check multiple common locations
|
|
11
|
+
ENV_FILES=(".env" "$(dirname "$0")/../../../../.env" "$HOME/.synth-ai/.env")
|
|
12
|
+
for env_file in "${ENV_FILES[@]}"; do
|
|
13
|
+
if [ -f "$env_file" ]; then
|
|
14
|
+
echo "📝 Loading environment from: $env_file"
|
|
15
|
+
set -a # automatically export all variables
|
|
16
|
+
source "$env_file"
|
|
17
|
+
set +a
|
|
18
|
+
break
|
|
19
|
+
fi
|
|
20
|
+
done
|
|
21
|
+
|
|
22
|
+
# Set up environment variables
|
|
23
|
+
export ENVIRONMENT_API_KEY="${ENVIRONMENT_API_KEY:-$(python -c 'import secrets; print(secrets.token_urlsafe(32))')}"
|
|
24
|
+
export GROQ_API_KEY="${GROQ_API_KEY}"
|
|
25
|
+
|
|
26
|
+
# Check for required env vars
|
|
27
|
+
if [ -z "$GROQ_API_KEY" ]; then
|
|
28
|
+
echo "❌ Error: GROQ_API_KEY not set"
|
|
29
|
+
echo "Please set it: export GROQ_API_KEY=your_key"
|
|
30
|
+
exit 1
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
echo "✅ ENVIRONMENT_API_KEY: ${ENVIRONMENT_API_KEY:0:20}..."
|
|
34
|
+
echo "✅ GROQ_API_KEY: ${GROQ_API_KEY:0:20}..."
|
|
35
|
+
|
|
36
|
+
# Navigate to repo root
|
|
37
|
+
cd "$(dirname "$0")/../../.."
|
|
38
|
+
|
|
39
|
+
echo ""
|
|
40
|
+
echo "📦 Installing dependencies..."
|
|
41
|
+
uv pip install -e . --quiet || true
|
|
42
|
+
|
|
43
|
+
echo ""
|
|
44
|
+
echo "🏃 Starting Banking77 task app on http://127.0.0.1:8102"
|
|
45
|
+
echo "Press Ctrl+C to stop"
|
|
46
|
+
echo ""
|
|
47
|
+
|
|
48
|
+
# Run the task app
|
|
49
|
+
python -m examples.task_apps.banking77.banking77_task_app \
|
|
50
|
+
--host 0.0.0.0 \
|
|
51
|
+
--port 8102 \
|
|
52
|
+
--reload
|
|
53
|
+
|
|
54
|
+
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Banking77 baseline file for intent classification evaluation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict
|
|
6
|
+
|
|
7
|
+
from datasets import load_dataset
|
|
8
|
+
|
|
9
|
+
from synth_ai.baseline import BaselineConfig, BaselineTaskRunner, DataSplit, TaskResult
|
|
10
|
+
from synth_ai.inference import InferenceClient
|
|
11
|
+
import os
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Load dataset once at module level
|
|
16
|
+
_dataset = None
|
|
17
|
+
_label_names = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _load_dataset():
|
|
21
|
+
"""Load Banking77 dataset."""
|
|
22
|
+
global _dataset, _label_names
|
|
23
|
+
if _dataset is None:
|
|
24
|
+
try:
|
|
25
|
+
_dataset = load_dataset("PolyAI/banking77")
|
|
26
|
+
except Exception:
|
|
27
|
+
# Fallback: try without org prefix
|
|
28
|
+
_dataset = load_dataset("banking77")
|
|
29
|
+
_label_names = _dataset["train"].features["label"].names
|
|
30
|
+
return _dataset, _label_names
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Banking77TaskRunner(BaselineTaskRunner):
|
|
34
|
+
"""Task runner for Banking77 intent classification."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, policy_config: Dict[str, Any], env_config: Dict[str, Any]):
|
|
37
|
+
super().__init__(policy_config, env_config)
|
|
38
|
+
|
|
39
|
+
# Load dataset
|
|
40
|
+
self.dataset, self.label_names = _load_dataset()
|
|
41
|
+
|
|
42
|
+
# Store config for inference
|
|
43
|
+
self.model = policy_config["model"]
|
|
44
|
+
self.temperature = policy_config.get("temperature", 0.0)
|
|
45
|
+
self.max_tokens = policy_config.get("max_tokens", 128)
|
|
46
|
+
self.inference_url = policy_config.get("inference_url")
|
|
47
|
+
|
|
48
|
+
# Tool definition
|
|
49
|
+
self.tool = {
|
|
50
|
+
"type": "function",
|
|
51
|
+
"function": {
|
|
52
|
+
"name": "banking77_classify",
|
|
53
|
+
"description": "Classify a banking query into an intent",
|
|
54
|
+
"parameters": {
|
|
55
|
+
"type": "object",
|
|
56
|
+
"properties": {
|
|
57
|
+
"label": {
|
|
58
|
+
"type": "string",
|
|
59
|
+
"enum": self.label_names,
|
|
60
|
+
"description": "The intent label",
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
"required": ["label"],
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async def run_task(self, seed: int) -> TaskResult:
|
|
69
|
+
"""Run a single Banking77 classification task."""
|
|
70
|
+
|
|
71
|
+
# Get split
|
|
72
|
+
split = self.env_config.get("split", "train")
|
|
73
|
+
|
|
74
|
+
# Get example from dataset
|
|
75
|
+
example = self.dataset[split][seed]
|
|
76
|
+
|
|
77
|
+
# Build prompt
|
|
78
|
+
system_prompt = f"""You are an expert banking assistant that classifies customer queries.
|
|
79
|
+
Given a customer message, respond with exactly one intent label using the tool call.
|
|
80
|
+
|
|
81
|
+
Valid intents: {', '.join(self.label_names)}"""
|
|
82
|
+
|
|
83
|
+
user_prompt = f"Customer Query: {example['text']}\n\nClassify this query."
|
|
84
|
+
|
|
85
|
+
# Run inference
|
|
86
|
+
messages = [
|
|
87
|
+
{"role": "system", "content": system_prompt},
|
|
88
|
+
{"role": "user", "content": user_prompt},
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
# Use InferenceClient if URL provided, otherwise use OpenAI-compatible API
|
|
92
|
+
if self.inference_url and self.inference_url.startswith("http"):
|
|
93
|
+
api_key = os.getenv("SYNTH_API_KEY") or os.getenv("OPENAI_API_KEY") or ""
|
|
94
|
+
base_url = self.inference_url.rstrip("/")
|
|
95
|
+
if not base_url.endswith("/api"):
|
|
96
|
+
base_url = f"{base_url}/api" if "/api" not in base_url else base_url
|
|
97
|
+
client = InferenceClient(base_url=base_url, api_key=api_key)
|
|
98
|
+
response = await client.create_chat_completion(
|
|
99
|
+
model=self.model,
|
|
100
|
+
messages=messages,
|
|
101
|
+
tools=[self.tool],
|
|
102
|
+
tool_choice={"type": "function", "function": {"name": "banking77_classify"}},
|
|
103
|
+
temperature=self.temperature,
|
|
104
|
+
max_tokens=self.max_tokens,
|
|
105
|
+
)
|
|
106
|
+
else:
|
|
107
|
+
# Use OpenAI/Groq directly
|
|
108
|
+
# Check if model starts with groq: prefix
|
|
109
|
+
model_name = self.model
|
|
110
|
+
use_groq = model_name.startswith("groq:")
|
|
111
|
+
if use_groq:
|
|
112
|
+
model_name = model_name[5:] # Remove "groq:" prefix
|
|
113
|
+
|
|
114
|
+
api_key = os.getenv("GROQ_API_KEY") if use_groq else os.getenv("OPENAI_API_KEY") or ""
|
|
115
|
+
base_url = "https://api.groq.com/openai/v1" if use_groq else "https://api.openai.com/v1"
|
|
116
|
+
async with httpx.AsyncClient() as http_client:
|
|
117
|
+
resp = await http_client.post(
|
|
118
|
+
f"{base_url}/chat/completions",
|
|
119
|
+
json={
|
|
120
|
+
"model": model_name,
|
|
121
|
+
"messages": messages,
|
|
122
|
+
"tools": [self.tool],
|
|
123
|
+
"tool_choice": {"type": "function", "function": {"name": "banking77_classify"}},
|
|
124
|
+
"temperature": self.temperature,
|
|
125
|
+
"max_tokens": self.max_tokens,
|
|
126
|
+
},
|
|
127
|
+
headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
|
|
128
|
+
)
|
|
129
|
+
response = resp.json()
|
|
130
|
+
|
|
131
|
+
# Extract prediction
|
|
132
|
+
predicted_label = ""
|
|
133
|
+
tool_calls = []
|
|
134
|
+
if "choices" in response and len(response["choices"]) > 0:
|
|
135
|
+
message = response["choices"][0].get("message", {})
|
|
136
|
+
tool_calls = message.get("tool_calls", [])
|
|
137
|
+
elif "tool_calls" in response:
|
|
138
|
+
tool_calls = response["tool_calls"]
|
|
139
|
+
|
|
140
|
+
if tool_calls:
|
|
141
|
+
# Handle both string and dict arguments
|
|
142
|
+
args = tool_calls[0]["function"].get("arguments", "")
|
|
143
|
+
if isinstance(args, str):
|
|
144
|
+
import json
|
|
145
|
+
args = json.loads(args)
|
|
146
|
+
predicted_label = args.get("label", "") if isinstance(args, dict) else ""
|
|
147
|
+
|
|
148
|
+
# Evaluate
|
|
149
|
+
expected_label = self.label_names[example["label"]]
|
|
150
|
+
correct = predicted_label == expected_label
|
|
151
|
+
|
|
152
|
+
return TaskResult(
|
|
153
|
+
seed=seed,
|
|
154
|
+
success=True,
|
|
155
|
+
outcome_reward=1.0 if correct else 0.0,
|
|
156
|
+
total_steps=1,
|
|
157
|
+
metadata={
|
|
158
|
+
"query": example["text"],
|
|
159
|
+
"expected": expected_label,
|
|
160
|
+
"predicted": predicted_label,
|
|
161
|
+
"correct": correct,
|
|
162
|
+
"split": split,
|
|
163
|
+
},
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# Define baseline config
|
|
168
|
+
# Note: We need to load the dataset first to get the label names
|
|
169
|
+
_load_dataset()
|
|
170
|
+
gepa_baseline = BaselineConfig(
|
|
171
|
+
baseline_id="gepa",
|
|
172
|
+
name="GEPA - Banking77 Intent Classification",
|
|
173
|
+
description="Banking77 intent classification baseline for prompt optimization experiments",
|
|
174
|
+
task_runner=Banking77TaskRunner,
|
|
175
|
+
splits={
|
|
176
|
+
"train": DataSplit(
|
|
177
|
+
name="train",
|
|
178
|
+
seeds=list(range(min(10000, len(_dataset["train"]))) if _dataset else range(10000)),
|
|
179
|
+
),
|
|
180
|
+
"val": DataSplit(
|
|
181
|
+
name="val",
|
|
182
|
+
seeds=list(range(min(1000, len(_dataset["test"]))) if _dataset else range(1000)),
|
|
183
|
+
),
|
|
184
|
+
"test": DataSplit(
|
|
185
|
+
name="test",
|
|
186
|
+
seeds=list(range(min(3000, len(_dataset["test"]))) if _dataset else range(3000)),
|
|
187
|
+
),
|
|
188
|
+
},
|
|
189
|
+
default_policy_config={
|
|
190
|
+
"model": "groq:llama-3.1-70b-versatile",
|
|
191
|
+
"temperature": 0.0,
|
|
192
|
+
"max_tokens": 128,
|
|
193
|
+
},
|
|
194
|
+
default_env_config={
|
|
195
|
+
"split": "train",
|
|
196
|
+
},
|
|
197
|
+
metadata={
|
|
198
|
+
"dataset": "PolyAI/banking77",
|
|
199
|
+
"num_classes": 77,
|
|
200
|
+
"task_type": "classification",
|
|
201
|
+
},
|
|
202
|
+
tags=["classification", "nlp", "intent", "blog-post"],
|
|
203
|
+
)
|
|
204
|
+
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Example script showing how to query prompt learning job results.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
python query_prompts_example.py pl_9c58b711c2644083
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
from pprint import pprint
|
|
11
|
+
|
|
12
|
+
from synth_ai.learning import get_prompts, get_prompt_text, get_scoring_summary
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main():
|
|
16
|
+
if len(sys.argv) < 2:
|
|
17
|
+
print("Usage: python query_prompts_example.py <job_id>")
|
|
18
|
+
print("Example: python query_prompts_example.py pl_9c58b711c2644083")
|
|
19
|
+
sys.exit(1)
|
|
20
|
+
|
|
21
|
+
job_id = sys.argv[1]
|
|
22
|
+
|
|
23
|
+
# Get credentials from environment
|
|
24
|
+
base_url = os.getenv("BACKEND_BASE_URL", "http://localhost:8000")
|
|
25
|
+
api_key = os.getenv("SYNTH_API_KEY")
|
|
26
|
+
|
|
27
|
+
if not api_key:
|
|
28
|
+
print("Error: SYNTH_API_KEY environment variable not set")
|
|
29
|
+
sys.exit(1)
|
|
30
|
+
|
|
31
|
+
print(f"Querying job: {job_id}")
|
|
32
|
+
print(f"Backend: {base_url}")
|
|
33
|
+
print("=" * 80)
|
|
34
|
+
|
|
35
|
+
# Get all prompts and metadata
|
|
36
|
+
print("\n📊 Fetching prompt results...")
|
|
37
|
+
results = get_prompts(job_id, base_url, api_key)
|
|
38
|
+
|
|
39
|
+
# Print best score
|
|
40
|
+
if results.best_score is not None:
|
|
41
|
+
print(f"\n🏆 Best Score: {results.best_score:.3f} ({results.best_score * 100:.1f}%)")
|
|
42
|
+
|
|
43
|
+
# Print top-K prompts with scores
|
|
44
|
+
top_prompts = results.top_prompts
|
|
45
|
+
if top_prompts:
|
|
46
|
+
print(f"\n📝 Top {len(top_prompts)} Prompts:")
|
|
47
|
+
print("=" * 80)
|
|
48
|
+
for prompt_info in sorted(top_prompts, key=lambda p: p.get("rank", 999)):
|
|
49
|
+
rank = prompt_info["rank"]
|
|
50
|
+
train_accuracy = prompt_info.get("train_accuracy")
|
|
51
|
+
val_accuracy = prompt_info.get("val_accuracy")
|
|
52
|
+
|
|
53
|
+
print(f"\nRank #{rank}:")
|
|
54
|
+
if train_accuracy is not None:
|
|
55
|
+
print(f" Train Accuracy: {train_accuracy:.3f} ({train_accuracy * 100:.1f}%)")
|
|
56
|
+
if val_accuracy is not None:
|
|
57
|
+
print(f" Val Accuracy: {val_accuracy:.3f} ({val_accuracy * 100:.1f}%)")
|
|
58
|
+
print(f" Prompt Text:")
|
|
59
|
+
print(" " + "-" * 76)
|
|
60
|
+
full_text = prompt_info.get("full_text", "")
|
|
61
|
+
for line in full_text.split("\n"):
|
|
62
|
+
print(f" {line}")
|
|
63
|
+
print(" " + "-" * 76)
|
|
64
|
+
|
|
65
|
+
# Get scoring summary
|
|
66
|
+
print("\n📈 Scoring Summary:")
|
|
67
|
+
print("=" * 80)
|
|
68
|
+
summary = get_scoring_summary(job_id, base_url, api_key)
|
|
69
|
+
|
|
70
|
+
print(f"Best Train Accuracy: {summary['best_train_accuracy']:.3f} ({summary['best_train_accuracy'] * 100:.1f}%)")
|
|
71
|
+
if summary['best_val_accuracy']:
|
|
72
|
+
print(f"Best Val Accuracy: {summary['best_val_accuracy']:.3f} ({summary['best_val_accuracy'] * 100:.1f}%)")
|
|
73
|
+
print(f"Mean Train Accuracy: {summary['mean_train_accuracy']:.3f} ({summary['mean_train_accuracy'] * 100:.1f}%)")
|
|
74
|
+
print(f"Candidates Tried: {summary['num_candidates_tried']}")
|
|
75
|
+
print(f"Frontier Candidates: {summary['num_frontier_candidates']}")
|
|
76
|
+
|
|
77
|
+
print(f"\nScore Distribution:")
|
|
78
|
+
for bin_range, count in summary['score_distribution'].items():
|
|
79
|
+
bar = "█" * count
|
|
80
|
+
print(f" {bin_range}: {count:3d} {bar}")
|
|
81
|
+
|
|
82
|
+
# Quick access to best prompt text only
|
|
83
|
+
print("\n💡 Quick access to best prompt:")
|
|
84
|
+
print("=" * 80)
|
|
85
|
+
best_text = get_prompt_text(job_id, base_url, api_key, rank=1)
|
|
86
|
+
if best_text:
|
|
87
|
+
print(best_text)
|
|
88
|
+
else:
|
|
89
|
+
print("Best prompt text not available yet (job may still be running)")
|
|
90
|
+
|
|
91
|
+
print("\n" + "=" * 80)
|
|
92
|
+
print("✅ Query complete!")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
if __name__ == "__main__":
|
|
96
|
+
main()
|
|
97
|
+
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Run GEPA optimization for Banking77 against the backend
|
|
3
|
+
|
|
4
|
+
set -e
|
|
5
|
+
|
|
6
|
+
echo "🧬 Running GEPA on Banking77"
|
|
7
|
+
echo "============================="
|
|
8
|
+
|
|
9
|
+
# Navigate to repo root first
|
|
10
|
+
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
|
11
|
+
cd "$REPO_ROOT"
|
|
12
|
+
|
|
13
|
+
# Load environment variables from .env file if it exists
|
|
14
|
+
_load_env_file() {
|
|
15
|
+
local env_file="$1"
|
|
16
|
+
if [ -f "$env_file" ]; then
|
|
17
|
+
echo "📝 Loading environment variables from $env_file..."
|
|
18
|
+
# Only export lines that look like KEY=VALUE (handles comments and empty lines)
|
|
19
|
+
while IFS= read -r line || [ -n "$line" ]; do
|
|
20
|
+
# Skip comments and empty lines
|
|
21
|
+
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
|
22
|
+
[[ -z "${line// }" ]] && continue
|
|
23
|
+
# Only export if it looks like KEY=VALUE
|
|
24
|
+
if [[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]]; then
|
|
25
|
+
export "$line" 2>/dev/null || true
|
|
26
|
+
fi
|
|
27
|
+
done < "$env_file"
|
|
28
|
+
fi
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
_load_env_file "$REPO_ROOT/.env"
|
|
32
|
+
_load_env_file "$REPO_ROOT/examples/rl/.env"
|
|
33
|
+
|
|
34
|
+
# Check for required environment variables
|
|
35
|
+
if [ -z "$SYNTH_API_KEY" ]; then
|
|
36
|
+
echo "❌ Error: SYNTH_API_KEY not set"
|
|
37
|
+
echo "Please get your API key from the backend and set it:"
|
|
38
|
+
echo " export SYNTH_API_KEY=your_key"
|
|
39
|
+
exit 1
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
if [ -z "$ENVIRONMENT_API_KEY" ]; then
|
|
43
|
+
echo "❌ Error: ENVIRONMENT_API_KEY not set"
|
|
44
|
+
echo "Please set the same key used when deploying the task app:"
|
|
45
|
+
echo " export ENVIRONMENT_API_KEY=your_key"
|
|
46
|
+
exit 1
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
if [ -z "$GROQ_API_KEY" ]; then
|
|
50
|
+
echo "❌ Error: GROQ_API_KEY not set"
|
|
51
|
+
echo "Please set your Groq API key:"
|
|
52
|
+
echo " export GROQ_API_KEY=your_key"
|
|
53
|
+
exit 1
|
|
54
|
+
fi
|
|
55
|
+
|
|
56
|
+
# Default to localhost backend if not specified
|
|
57
|
+
BACKEND_URL="${BACKEND_BASE_URL:-http://localhost:8000}"
|
|
58
|
+
|
|
59
|
+
echo "✅ SYNTH_API_KEY: ${SYNTH_API_KEY:0:20}..."
|
|
60
|
+
echo "✅ ENVIRONMENT_API_KEY: ${ENVIRONMENT_API_KEY:0:20}..."
|
|
61
|
+
echo "✅ GROQ_API_KEY: ${GROQ_API_KEY:0:20}..."
|
|
62
|
+
echo "✅ Backend URL: $BACKEND_URL"
|
|
63
|
+
echo ""
|
|
64
|
+
|
|
65
|
+
# Navigate to repo root
|
|
66
|
+
cd "$(dirname "$0")/../../.."
|
|
67
|
+
|
|
68
|
+
# Check if task app is running
|
|
69
|
+
echo "🔍 Checking if Banking77 task app is running on http://127.0.0.1:8102..."
|
|
70
|
+
if ! curl -s -f -H "X-API-Key: $ENVIRONMENT_API_KEY" http://127.0.0.1:8102/health > /dev/null 2>&1; then
|
|
71
|
+
echo "❌ Error: Banking77 task app is not running on http://127.0.0.1:8102"
|
|
72
|
+
echo ""
|
|
73
|
+
echo "Please start it first:"
|
|
74
|
+
echo " ./examples/blog_posts/gepa/deploy_banking77_task_app.sh"
|
|
75
|
+
echo ""
|
|
76
|
+
echo "Or in another terminal:"
|
|
77
|
+
echo " cd $(pwd)"
|
|
78
|
+
echo " uvx synth-ai deploy banking77 --runtime uvicorn --port 8102"
|
|
79
|
+
exit 1
|
|
80
|
+
fi
|
|
81
|
+
echo "✅ Task app is healthy"
|
|
82
|
+
echo ""
|
|
83
|
+
|
|
84
|
+
# Check backend connection
|
|
85
|
+
echo "🔍 Checking backend connection to $BACKEND_URL..."
|
|
86
|
+
if ! curl -s -f "$BACKEND_URL/api/health" > /dev/null 2>&1; then
|
|
87
|
+
echo "⚠️ Warning: Cannot connect to backend at $BACKEND_URL"
|
|
88
|
+
echo "Make sure the backend is running."
|
|
89
|
+
read -p "Continue anyway? (y/N) " -n 1 -r
|
|
90
|
+
echo
|
|
91
|
+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
92
|
+
exit 1
|
|
93
|
+
fi
|
|
94
|
+
else
|
|
95
|
+
echo "✅ Backend is healthy"
|
|
96
|
+
fi
|
|
97
|
+
echo ""
|
|
98
|
+
|
|
99
|
+
echo "🚀 Starting GEPA training..."
|
|
100
|
+
echo "Config: examples/blog_posts/gepa/configs/banking77_gepa_local.toml"
|
|
101
|
+
echo ""
|
|
102
|
+
|
|
103
|
+
# Run the training
|
|
104
|
+
uvx synth-ai train \
|
|
105
|
+
--type prompt_learning \
|
|
106
|
+
--config examples/blog_posts/gepa/configs/banking77_gepa_local.toml \
|
|
107
|
+
--backend "$BACKEND_URL" \
|
|
108
|
+
--poll
|
|
109
|
+
|
|
110
|
+
echo ""
|
|
111
|
+
echo "✅ GEPA training complete!"
|
|
112
|
+
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Run GEPA optimisation for the multi-step Banking77 pipeline against the backend
|
|
3
|
+
|
|
4
|
+
set -euo pipefail
|
|
5
|
+
|
|
6
|
+
echo "🧬 Running GEPA on Banking77 Pipeline"
|
|
7
|
+
echo "======================================"
|
|
8
|
+
|
|
9
|
+
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
|
10
|
+
cd "$REPO_ROOT"
|
|
11
|
+
|
|
12
|
+
SAVED_BACKEND_BASE_URL="${BACKEND_BASE_URL:-}"
|
|
13
|
+
SAVED_SYNTH_BASE_URL="${SYNTH_BASE_URL:-}"
|
|
14
|
+
|
|
15
|
+
_load_env_file() {
|
|
16
|
+
local env_file="$1"
|
|
17
|
+
if [ -f "$env_file" ]; then
|
|
18
|
+
echo "📝 Loading environment variables from $env_file..."
|
|
19
|
+
while IFS= read -r line || [ -n "$line" ]; do
|
|
20
|
+
# Skip comments and empty lines
|
|
21
|
+
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
|
22
|
+
[[ -z "${line// }" ]] && continue
|
|
23
|
+
# Only process lines with '=' character
|
|
24
|
+
[[ ! "$line" =~ = ]] && continue
|
|
25
|
+
|
|
26
|
+
if [[ "$line" =~ ^[[:space:]]*BACKEND_BASE_URL= ]]; then
|
|
27
|
+
if [ -z "$SAVED_BACKEND_BASE_URL" ]; then
|
|
28
|
+
export "$line" 2>/dev/null || true
|
|
29
|
+
else
|
|
30
|
+
echo " ⚠️ Skipping BACKEND_BASE_URL from .env (using environment value)"
|
|
31
|
+
fi
|
|
32
|
+
elif [[ "$line" =~ ^[[:space:]]*SYNTH_BASE_URL= ]]; then
|
|
33
|
+
if [ -z "$SAVED_SYNTH_BASE_URL" ]; then
|
|
34
|
+
export "$line" 2>/dev/null || true
|
|
35
|
+
else
|
|
36
|
+
echo " ⚠️ Skipping SYNTH_BASE_URL from .env (using environment value)"
|
|
37
|
+
fi
|
|
38
|
+
elif [[ "$line" =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*= ]]; then
|
|
39
|
+
export "$line" 2>/dev/null || true
|
|
40
|
+
fi
|
|
41
|
+
done < "$env_file"
|
|
42
|
+
fi
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
_load_env_file "$REPO_ROOT/.env"
|
|
46
|
+
_load_env_file "$REPO_ROOT/examples/rl/.env"
|
|
47
|
+
|
|
48
|
+
if [ -n "$SAVED_BACKEND_BASE_URL" ]; then
|
|
49
|
+
export BACKEND_BASE_URL="$SAVED_BACKEND_BASE_URL"
|
|
50
|
+
echo "✅ Using BACKEND_BASE_URL from environment: $BACKEND_BASE_URL"
|
|
51
|
+
fi
|
|
52
|
+
if [ -n "$SAVED_SYNTH_BASE_URL" ]; then
|
|
53
|
+
export SYNTH_BASE_URL="$SAVED_SYNTH_BASE_URL"
|
|
54
|
+
echo "✅ Using SYNTH_BASE_URL from environment: $SYNTH_BASE_URL"
|
|
55
|
+
fi
|
|
56
|
+
|
|
57
|
+
if [ -z "${SYNTH_API_KEY:-}" ]; then
|
|
58
|
+
echo "❌ Error: SYNTH_API_KEY not set"
|
|
59
|
+
exit 1
|
|
60
|
+
fi
|
|
61
|
+
|
|
62
|
+
if [ -z "${ENVIRONMENT_API_KEY:-}" ]; then
|
|
63
|
+
echo "❌ Error: ENVIRONMENT_API_KEY not set"
|
|
64
|
+
exit 1
|
|
65
|
+
fi
|
|
66
|
+
|
|
67
|
+
if [ -z "${GROQ_API_KEY:-}" ] && [ -z "${OPENAI_API_KEY:-}" ]; then
|
|
68
|
+
echo "⚠️ Warning: Neither GROQ_API_KEY nor OPENAI_API_KEY is set."
|
|
69
|
+
echo "The policy defaults to Groq-hosted OSS models. Set GROQ_API_KEY to avoid failures."
|
|
70
|
+
fi
|
|
71
|
+
|
|
72
|
+
if [ -z "${BACKEND_BASE_URL:-}" ]; then
|
|
73
|
+
BACKEND_BASE_URL="http://localhost:8000"
|
|
74
|
+
fi
|
|
75
|
+
BACKEND_URL="$BACKEND_BASE_URL"
|
|
76
|
+
BACKEND_URL_NO_API="${BACKEND_URL%/api}"
|
|
77
|
+
|
|
78
|
+
echo ""
|
|
79
|
+
echo "🔧 Debug Info:"
|
|
80
|
+
echo " BACKEND_BASE_URL current: $BACKEND_BASE_URL"
|
|
81
|
+
echo " BACKEND_URL: $BACKEND_URL"
|
|
82
|
+
echo ""
|
|
83
|
+
|
|
84
|
+
echo "✅ SYNTH_API_KEY: ${SYNTH_API_KEY:0:20}..."
|
|
85
|
+
echo "✅ ENVIRONMENT_API_KEY: ${ENVIRONMENT_API_KEY:0:20}..."
|
|
86
|
+
if [ -n "${GROQ_API_KEY:-}" ]; then
|
|
87
|
+
echo "✅ GROQ_API_KEY: ${GROQ_API_KEY:0:20}..."
|
|
88
|
+
fi
|
|
89
|
+
if [ -n "${OPENAI_API_KEY:-}" ]; then
|
|
90
|
+
echo "✅ OPENAI_API_KEY: ${OPENAI_API_KEY:0:20}..."
|
|
91
|
+
fi
|
|
92
|
+
echo "✅ Backend URL: $BACKEND_URL"
|
|
93
|
+
echo ""
|
|
94
|
+
|
|
95
|
+
CONFIG_PATH="examples/blog_posts/gepa/configs/banking77_pipeline_gepa_local.toml"
|
|
96
|
+
|
|
97
|
+
# ALWAYS read task_app_url from TOML (never use any pre-existing shell variable)
|
|
98
|
+
# Override only if TASK_APP_URL_OVERRIDE is explicitly set
|
|
99
|
+
if [ -n "${TASK_APP_URL_OVERRIDE:-}" ]; then
|
|
100
|
+
TASK_APP_URL="$TASK_APP_URL_OVERRIDE"
|
|
101
|
+
echo "📝 Using OVERRIDE task app URL: $TASK_APP_URL"
|
|
102
|
+
else
|
|
103
|
+
# Extract task_app_url from TOML using grep/sed (no Python dependencies)
|
|
104
|
+
# Pattern: task_app_url = "https://..."
|
|
105
|
+
TASK_APP_URL="$(grep "^task_app_url" "$CONFIG_PATH" | sed -E 's/^[^=]*=[[:space:]]*"([^"]*)".*/\1/' | head -1)"
|
|
106
|
+
if [ -z "$TASK_APP_URL" ]; then
|
|
107
|
+
echo "❌ ERROR: task_app_url not found in $CONFIG_PATH" >&2
|
|
108
|
+
echo " Please ensure the config file contains: task_app_url = \"...\"" >&2
|
|
109
|
+
exit 1
|
|
110
|
+
fi
|
|
111
|
+
echo "📝 Task app URL from TOML: $TASK_APP_URL"
|
|
112
|
+
fi
|
|
113
|
+
echo ""
|
|
114
|
+
|
|
115
|
+
echo "🔍 Checking if Banking77 pipeline task app is running on ${TASK_APP_URL}..."
|
|
116
|
+
if ! curl -s -f -H "X-API-Key: $ENVIRONMENT_API_KEY" "$TASK_APP_URL/health" > /dev/null 2>&1; then
|
|
117
|
+
cat <<EOF
|
|
118
|
+
❌ Error: Banking77 pipeline task app is not running on ${TASK_APP_URL}
|
|
119
|
+
|
|
120
|
+
Start it with:
|
|
121
|
+
uvx synth-ai deploy banking77-pipeline --runtime uvicorn --port 8112 --env-file .env --follow
|
|
122
|
+
# or deploy to Modal dev: modal deploy --env dev examples/task_apps/banking77_pipeline/deploy_wrapper.py
|
|
123
|
+
EOF
|
|
124
|
+
exit 1
|
|
125
|
+
fi
|
|
126
|
+
echo "✅ Pipeline task app is healthy"
|
|
127
|
+
echo ""
|
|
128
|
+
|
|
129
|
+
echo "🔍 Checking backend connection to $BACKEND_URL..."
|
|
130
|
+
if ! curl -s -f "$BACKEND_URL_NO_API/api/health" > /dev/null 2>&1; then
|
|
131
|
+
echo "⚠️ Warning: Cannot connect to backend at $BACKEND_URL"
|
|
132
|
+
read -p "Continue anyway? (y/N) " -n 1 -r
|
|
133
|
+
echo
|
|
134
|
+
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
135
|
+
exit 1
|
|
136
|
+
fi
|
|
137
|
+
else
|
|
138
|
+
echo "✅ Backend is healthy"
|
|
139
|
+
fi
|
|
140
|
+
echo ""
|
|
141
|
+
|
|
142
|
+
echo "🚀 Starting GEPA training..."
|
|
143
|
+
echo "Config: $CONFIG_PATH"
|
|
144
|
+
echo ""
|
|
145
|
+
echo "Multi-Stage Flow:"
|
|
146
|
+
echo " - Stages: classifier → calibrator"
|
|
147
|
+
echo " - Training seeds: [0-14]"
|
|
148
|
+
echo " - Validation seeds: [15-19]"
|
|
149
|
+
echo " - Test pool: [40-49]"
|
|
150
|
+
echo ""
|
|
151
|
+
|
|
152
|
+
export BACKEND_BASE_URL="$BACKEND_URL"
|
|
153
|
+
export SYNTH_BASE_URL="$BACKEND_URL"
|
|
154
|
+
|
|
155
|
+
uvx synth-ai train \
|
|
156
|
+
--type prompt_learning \
|
|
157
|
+
--config "$CONFIG_PATH" \
|
|
158
|
+
--backend "$BACKEND_URL" \
|
|
159
|
+
--poll
|
|
160
|
+
|
|
161
|
+
echo ""
|
|
162
|
+
echo "✅ GEPA pipeline training complete!"
|
|
163
|
+
|