synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

examples/blog_posts/gepa/configs/hotpotqa_mipro_local.toml

@@ -0,0 +1,53 @@
+# MIPROv2 Prompt Learning for HotpotQA
+# Local backend configuration targeting the HotpotQA task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8110"
+task_app_id = "hotpotqa"
+
+# Seeds used during online optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hotpotqa_chain"
+name = "HotpotQA Multi-Hop Reasoning"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are a research assistant that answers multi-hop questions. Read the supporting passages carefully and articulate the final answer plus a short justification. Use the format:\nAnswer: ...\nSupport: ..."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Question: {question}\n\nPassages:\n{context}\n\nProvide the final answer and cite the supporting facts."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+question = "REQUIRED"
+context = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hotpotqa-mipro"
+
+[prompt_learning.mipro]
+env_name = "hotpotqa"
+num_iterations = 20
+num_evaluations_per_iteration = 8
+batch_size = 8
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+meta_model_inference_url = "https://api.openai.com/v1"
+few_shot_score_threshold = 0.75
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/configs/hover_gepa_local.toml

@@ -0,0 +1,59 @@
+# GEPA Prompt Learning for HoVer
+# Local backend configuration targeting the HoVer task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8112"
+task_app_id = "hover"
+
+# Seeds for online evaluation during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for final reporting
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hover_verification"
+name = "HoVer Claim Verification"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You verify Wikipedia claims. For each example decide whether the claim is SUPPORTED, REFUTED, or INSUFFICIENT and cite the sentences that justify the label."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Claim: {claim}\n\nEvidence:\n{evidence}\n\nRespond with the format:\nLabel: <SUPPORTED|REFUTED|INSUFFICIENT>\nRationale: <brief explanation citing evidence lines>."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+claim = "REQUIRED"
+evidence = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hover-gepa"
+
+[prompt_learning.gepa]
+env_name = "hover"
+initial_population_size = 24
+num_generations = 15
+mutation_rate = 0.33
+crossover_rate = 0.55
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.5
+children_per_generation = 12
+patience_generations = 5
+rollout_budget = 540
+archive_size = 36
+pareto_eps = 1e-6
+max_concurrent_rollouts = 20
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"
+mutation_llm_inference_url = "https://api.groq.com/openai/v1"

examples/blog_posts/gepa/configs/hover_gepa_qwen.toml

@@ -0,0 +1,36 @@
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-web-web.modal.run" # TODO: replace with HotpotQA task app URL
+task_app_id = "hotpotqa"
+
+# Seeds
+evaluation_seeds = [0,1,2,3,4,5,6,7,8,9]
+
+# Held-out validation
+validation_seeds = [10,11,12,13,14,15,16,17,18,19]
+validation_pool = "validation"
+validation_top_k = 3
+
+# Train split configuration
+[prompt_learning.env_config]
+pool = "train"
+
+# Policy model (synth Qwen via backend inference proxy)
+[prompt_learning.policy]
+provider = "synth"
+model = "Qwen/Qwen3-8B"
+# inference_url will be mapped to backend /api/inference/v1 by the optimizer
+
+# GEPA parameters (tune as needed)
+[prompt_learning.gepa]
+env_name = "hover"
+initial_population_size = 24
+num_generations = 6
+children_per_generation = 12
+minibatch_size = 10
+pareto_set_size = 32
+rollout_budget = 600
+max_concurrent_rollouts = 16
+mutation_llm_model = "openai/gpt-oss-120b"
+mutation_llm_provider = "groq"
+proposer_type = "dspy"

examples/blog_posts/gepa/configs/hover_mipro_local.toml

@@ -0,0 +1,53 @@
+# MIPROv2 Prompt Learning for HoVer
+# Local backend configuration targeting the HoVer task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8112"
+task_app_id = "hover"
+
+# Seeds explored during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds used for the final sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hover_verification"
+name = "HoVer Claim Verification"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are a fact-checking assistant. Review the evidence carefully and respond with SUPPORTED, REFUTED, or INSUFFICIENT along with a concise justification."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Claim: {claim}\n\nEvidence:\n{evidence}\n\nRespond with the format:\nLabel: <SUPPORTED|REFUTED|INSUFFICIENT>\nRationale: <brief explanation>."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+claim = "REQUIRED"
+evidence = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hover-mipro"
+
+[prompt_learning.mipro]
+env_name = "hover"
+num_iterations = 20
+num_evaluations_per_iteration = 6
+batch_size = 6
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+meta_model_inference_url = "https://api.openai.com/v1"
+few_shot_score_threshold = 0.8
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/configs/ifbench_gepa_local.toml

@@ -0,0 +1,59 @@
+# GEPA Prompt Learning for IFBench
+# Local backend configuration targeting the IFBench task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8111"
+task_app_id = "ifbench"
+
+# Candidate evaluation seeds during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out pool used for the final comparison sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "ifbench_following"
+name = "IFBench Instruction Following"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an obedient assistant that must follow instructions exactly. Ensure that every requirement is satisfied, avoid unsolicited commentary, and be explicit when information is missing."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Instruction: {instruction}\n\nInput: {input}\n\nProvide the response that best follows the instruction."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+instruction = "REQUIRED"
+input = "OPTIONAL"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "ifbench-gepa"
+
+[prompt_learning.gepa]
+env_name = "ifbench"
+initial_population_size = 24
+num_generations = 12
+mutation_rate = 0.3
+crossover_rate = 0.6
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.5
+children_per_generation = 12
+patience_generations = 4
+rollout_budget = 480
+archive_size = 32
+pareto_eps = 1e-6
+max_concurrent_rollouts = 20
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"
+mutation_llm_inference_url = "https://api.groq.com/openai/v1"

examples/blog_posts/gepa/configs/ifbench_gepa_qwen.toml

@@ -0,0 +1,36 @@
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-web-web.modal.run" # TODO: replace with HotpotQA task app URL
+task_app_id = "hotpotqa"
+
+# Seeds
+evaluation_seeds = [0,1,2,3,4,5,6,7,8,9]
+
+# Held-out validation
+validation_seeds = [10,11,12,13,14,15,16,17,18,19]
+validation_pool = "validation"
+validation_top_k = 3
+
+# Train split configuration
+[prompt_learning.env_config]
+pool = "train"
+
+# Policy model (synth Qwen via backend inference proxy)
+[prompt_learning.policy]
+provider = "synth"
+model = "Qwen/Qwen3-8B"
+# inference_url will be mapped to backend /api/inference/v1 by the optimizer
+
+# GEPA parameters (tune as needed)
+[prompt_learning.gepa]
+env_name = "ifbench"
+initial_population_size = 24
+num_generations = 6
+children_per_generation = 12
+minibatch_size = 10
+pareto_set_size = 32
+rollout_budget = 600
+max_concurrent_rollouts = 16
+mutation_llm_model = "openai/gpt-oss-120b"
+mutation_llm_provider = "groq"
+proposer_type = "dspy"

examples/blog_posts/gepa/configs/ifbench_mipro_local.toml

@@ -0,0 +1,53 @@
+# MIPROv2 Prompt Learning for IFBench
+# Local backend configuration targeting the IFBench task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8111"
+task_app_id = "ifbench"
+
+# Seeds evaluated during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final comparison
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "ifbench_following"
+name = "IFBench Instruction Following"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an obedient assistant that follows instructions exactly. Ensure that every constraint is satisfied and mention explicitly if something cannot be completed."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Instruction: {instruction}\n\nInput: {input}\n\nReturn the response that best follows the instruction."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+instruction = "REQUIRED"
+input = "OPTIONAL"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "ifbench-mipro"
+
+[prompt_learning.mipro]
+env_name = "ifbench"
+num_iterations = 16
+num_evaluations_per_iteration = 6
+batch_size = 6
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+meta_model_inference_url = "https://api.openai.com/v1"
+few_shot_score_threshold = 0.8
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/configs/pupa_gepa_local.toml

@@ -0,0 +1,60 @@
+# GEPA Prompt Learning for PUPA
+# Local backend configuration targeting the PUPA privacy-aware delegation task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8113"
+task_app_id = "pupa"
+
+# Seeds explored during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final evaluation
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "pupa_privacy"
+name = "PUPA Privacy-Constrained Delegation"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an assistant that must complete the task while honouring every privacy rule. Never reveal disallowed fields, always justify decisions, and explicitly state when data cannot be shared."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Task: {task}\n\nPrivacy Policy: {policy}\n\nRecords:\n{records}\n\nProduce the delegated plan or response while respecting the privacy policy."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+task = "REQUIRED"
+policy = "REQUIRED"
+records = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "pupa-gepa"
+
+[prompt_learning.gepa]
+env_name = "pupa"
+initial_population_size = 24
+num_generations = 15
+mutation_rate = 0.3
+crossover_rate = 0.6
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.6
+children_per_generation = 12
+patience_generations = 5
+rollout_budget = 540
+archive_size = 36
+pareto_eps = 1e-6
+max_concurrent_rollouts = 20
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"
+mutation_llm_inference_url = "https://api.groq.com/openai/v1"

examples/blog_posts/gepa/configs/pupa_mipro_local.toml

@@ -0,0 +1,54 @@
+# MIPROv2 Prompt Learning for PUPA
+# Local backend configuration targeting the PUPA privacy-aware delegation task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8113"
+task_app_id = "pupa"
+
+# Seeds evaluated during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "pupa_privacy"
+name = "PUPA Privacy-Constrained Delegation"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an assistant that must complete tasks without violating the privacy policy. Redact any forbidden attributes and justify refusals."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Task: {task}\n\nPrivacy Policy: {policy}\n\nRecords:\n{records}\n\nProvide the delegated plan or answer, ensuring compliance with the privacy policy."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+task = "REQUIRED"
+policy = "REQUIRED"
+records = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+inference_url = "https://api.groq.com/openai/v1"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "pupa-mipro"
+
+[prompt_learning.mipro]
+env_name = "pupa"
+num_iterations = 20
+num_evaluations_per_iteration = 6
+batch_size = 6
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+meta_model_inference_url = "https://api.openai.com/v1"
+few_shot_score_threshold = 0.85
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/deploy_banking77_task_app.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Deploy Banking77 task app locally for GEPA optimization
+
+set -e
+
+echo "🚀 Deploying Banking77 Task App..."
+echo "=================================="
+
+# Set up environment variables
+export ENVIRONMENT_API_KEY="${ENVIRONMENT_API_KEY:-$(python -c 'import secrets; print(secrets.token_urlsafe(32))')}"
+export GROQ_API_KEY="${GROQ_API_KEY}"
+
+# Check for required env vars
+if [ -z "$GROQ_API_KEY" ]; then
+    echo "❌ Error: GROQ_API_KEY not set"
+    echo "Please set it: export GROQ_API_KEY=your_key"
+    exit 1
+fi
+
+echo "✅ ENVIRONMENT_API_KEY: ${ENVIRONMENT_API_KEY:0:20}..."
+echo "✅ GROQ_API_KEY: ${GROQ_API_KEY:0:20}..."
+
+# Navigate to repo root
+cd "$(dirname "$0")/../../.."
+
+echo ""
+echo "📦 Installing dependencies..."
+uv pip install -e . --quiet || true
+
+echo ""
+echo "🏃 Starting Banking77 task app on http://127.0.0.1:8102"
+echo "Press Ctrl+C to stop"
+echo ""
+
+# Run the task app
+python -m examples.task_apps.banking77.banking77_task_app \
+    --host 0.0.0.0 \
+    --port 8102 \
+    --reload
+
+