synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

examples/blog_posts/gepa/configs/banking77_pipeline_gepa_test.toml

@@ -0,0 +1,96 @@
+# GEPA Prompt Learning for the Banking77 Two-Stage Pipeline (Test/Production)
+# This config optimizes both query_analyzer and classifier stages using genetic evolution
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-pipeline-web-web.modal.run"
+task_app_id = "banking77-pipeline"
+
+[prompt_learning.initial_prompt]
+id = "banking77_pipeline_baseline"
+name = "Banking77 Pipeline Baseline"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "Pipeline placeholder message. Actual stage instructions are provided via metadata.pipeline_modules."
+order = 0
+
+[prompt_learning.initial_prompt.metadata]
+# Define the stages in the pipeline
+pipeline_modules = [
+  { name = "query_analyzer", stage_id = "query_analyzer", module_id = "query_analyzer", instruction_text = "You analyze customer banking queries to extract key information. Identify the main topic and any specific details mentioned.", few_shots = [] },
+  { name = "classifier", stage_id = "classifier", module_id = "classifier", instruction_text = "You are an expert banking assistant that classifies customer queries into banking intents. Given a customer message, respond with exactly one intent label from the provided list using the `banking77_classify` tool.", few_shots = [] }
+]
+
+[prompt_learning.policy]
+inference_mode = "synth_hosted"
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 128
+policy_name = "banking77-pipeline"
+
+[prompt_learning.env_config]
+pool = "train"
+
+# GEPA-specific configuration
+[prompt_learning.gepa]
+env_name = "banking77_pipeline"
+rng_seed = 42
+proposer_type = "dspy"
+
+# Multi-stage module configuration (instruction-only for GEPA)
+# Each module defines constraints for its stage
+[[prompt_learning.gepa.modules]]
+module_id = "query_analyzer"
+max_instruction_slots = 2
+max_tokens = 512
+allowed_tools = []  # Analyzer doesn't use tools
+
+[[prompt_learning.gepa.modules]]
+module_id = "classifier"
+max_instruction_slots = 3
+max_tokens = 1024
+allowed_tools = ["banking77_classify"]  # Classifier must use this tool
+
+# Rollout configuration
+[prompt_learning.gepa.rollout]
+budget = 2000
+max_concurrent = 20
+minibatch_size = 8
+
+# Evaluation configuration
+[prompt_learning.gepa.evaluation]
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+validation_seeds = [15, 16, 17, 18, 19]
+test_pool = [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
+
+# Mutation configuration (LLM-guided)
+[prompt_learning.gepa.mutation]
+rate = 0.3
+llm_model = "llama3-groq-70b-8192-tool-use-preview"
+llm_provider = "groq"
+
+# Population/evolution configuration
+[prompt_learning.gepa.population]
+initial_size = 20
+num_generations = 10
+children_per_generation = 5
+crossover_rate = 0.5
+selection_pressure = 1.0
+patience_generations = 3
+
+# Archive/Pareto configuration
+[prompt_learning.gepa.archive]
+size = 64
+pareto_set_size = 64
+pareto_eps = 1e-6
+feedback_fraction = 0.5
+
+# Token/budget configuration
+[prompt_learning.gepa.token]
+max_limit = 50000
+counting_model = "gpt-4"
+enforce_pattern_limit = true
+max_spend_usd = 200.0
+

examples/blog_posts/gepa/configs/hotpotqa_gepa_local.toml

@@ -0,0 +1,57 @@
+# GEPA Prompt Learning for HotpotQA
+# Local backend configuration targeting the HotpotQA task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8110"
+task_app_id = "hotpotqa"
+
+# Seeds for online evaluation (episode IDs)
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out pool used for final evaluation
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hotpotqa_chain"
+name = "HotpotQA Multi-Hop Reasoning"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are a research assistant that answers multi-hop questions. Use the provided supporting passages to reason out the final answer. Reply with the format:\nAnswer: <short answer>\nSupport: <brief justification referencing the passages>."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Question: {question}\n\nPassages:\n{context}\n\nProvide the final answer and cite the relevant supporting facts."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+question = "REQUIRED"
+context = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hotpotqa-gepa"
+
+[prompt_learning.gepa]
+env_name = "hotpotqa"
+initial_population_size = 24
+num_generations = 15
+mutation_rate = 0.35
+crossover_rate = 0.55
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.5
+children_per_generation = 12
+patience_generations = 5
+rollout_budget = 600
+archive_size = 36
+pareto_eps = 1e-6
+max_concurrent_rollouts = 24
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"

examples/blog_posts/gepa/configs/hotpotqa_gepa_qwen.toml

@@ -0,0 +1,35 @@
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-web-web.modal.run" # TODO: replace with HotpotQA task app URL
+task_app_id = "hotpotqa"
+
+# Seeds
+evaluation_seeds = [0,1,2,3,4,5,6,7,8,9]
+
+# Held-out validation
+validation_seeds = [10,11,12,13,14,15,16,17,18,19]
+validation_pool = "validation"
+validation_top_k = 3
+
+# Train split configuration
+[prompt_learning.env_config]
+pool = "train"
+
+# Policy model (synth Qwen via backend inference proxy)
+[prompt_learning.policy]
+provider = "synth"
+model = "Qwen/Qwen3-8B"
+
+# GEPA parameters (tune as needed)
+[prompt_learning.gepa]
+env_name = "hotpotqa"
+initial_population_size = 24
+num_generations = 6
+children_per_generation = 12
+minibatch_size = 10
+pareto_set_size = 32
+rollout_budget = 600
+max_concurrent_rollouts = 16
+mutation_llm_model = "openai/gpt-oss-120b"
+mutation_llm_provider = "groq"
+proposer_type = "dspy"

examples/blog_posts/gepa/configs/hotpotqa_mipro_local.toml

@@ -0,0 +1,51 @@
+# MIPROv2 Prompt Learning for HotpotQA
+# Local backend configuration targeting the HotpotQA task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8110"
+task_app_id = "hotpotqa"
+
+# Seeds used during online optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hotpotqa_chain"
+name = "HotpotQA Multi-Hop Reasoning"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are a research assistant that answers multi-hop questions. Read the supporting passages carefully and articulate the final answer plus a short justification. Use the format:\nAnswer: ...\nSupport: ..."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Question: {question}\n\nPassages:\n{context}\n\nProvide the final answer and cite the supporting facts."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+question = "REQUIRED"
+context = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hotpotqa-mipro"
+
+[prompt_learning.mipro]
+env_name = "hotpotqa"
+num_iterations = 20
+num_evaluations_per_iteration = 8
+batch_size = 8
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+few_shot_score_threshold = 0.75
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/configs/hover_gepa_local.toml

@@ -0,0 +1,57 @@
+# GEPA Prompt Learning for HoVer
+# Local backend configuration targeting the HoVer task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8112"
+task_app_id = "hover"
+
+# Seeds for online evaluation during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for final reporting
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hover_verification"
+name = "HoVer Claim Verification"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You verify Wikipedia claims. For each example decide whether the claim is SUPPORTED, REFUTED, or INSUFFICIENT and cite the sentences that justify the label."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Claim: {claim}\n\nEvidence:\n{evidence}\n\nRespond with the format:\nLabel: <SUPPORTED|REFUTED|INSUFFICIENT>\nRationale: <brief explanation citing evidence lines>."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+claim = "REQUIRED"
+evidence = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hover-gepa"
+
+[prompt_learning.gepa]
+env_name = "hover"
+initial_population_size = 24
+num_generations = 15
+mutation_rate = 0.33
+crossover_rate = 0.55
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.5
+children_per_generation = 12
+patience_generations = 5
+rollout_budget = 540
+archive_size = 36
+pareto_eps = 1e-6
+max_concurrent_rollouts = 20
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"

examples/blog_posts/gepa/configs/hover_gepa_qwen.toml

@@ -0,0 +1,35 @@
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-web-web.modal.run" # TODO: replace with HotpotQA task app URL
+task_app_id = "hotpotqa"
+
+# Seeds
+evaluation_seeds = [0,1,2,3,4,5,6,7,8,9]
+
+# Held-out validation
+validation_seeds = [10,11,12,13,14,15,16,17,18,19]
+validation_pool = "validation"
+validation_top_k = 3
+
+# Train split configuration
+[prompt_learning.env_config]
+pool = "train"
+
+# Policy model (synth Qwen via backend inference proxy)
+[prompt_learning.policy]
+provider = "synth"
+model = "Qwen/Qwen3-8B"
+
+# GEPA parameters (tune as needed)
+[prompt_learning.gepa]
+env_name = "hover"
+initial_population_size = 24
+num_generations = 6
+children_per_generation = 12
+minibatch_size = 10
+pareto_set_size = 32
+rollout_budget = 600
+max_concurrent_rollouts = 16
+mutation_llm_model = "openai/gpt-oss-120b"
+mutation_llm_provider = "groq"
+proposer_type = "dspy"

examples/blog_posts/gepa/configs/hover_mipro_local.toml

@@ -0,0 +1,51 @@
+# MIPROv2 Prompt Learning for HoVer
+# Local backend configuration targeting the HoVer task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8112"
+task_app_id = "hover"
+
+# Seeds explored during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds used for the final sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "hover_verification"
+name = "HoVer Claim Verification"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are a fact-checking assistant. Review the evidence carefully and respond with SUPPORTED, REFUTED, or INSUFFICIENT along with a concise justification."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Claim: {claim}\n\nEvidence:\n{evidence}\n\nRespond with the format:\nLabel: <SUPPORTED|REFUTED|INSUFFICIENT>\nRationale: <brief explanation>."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+claim = "REQUIRED"
+evidence = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "hover-mipro"
+
+[prompt_learning.mipro]
+env_name = "hover"
+num_iterations = 20
+num_evaluations_per_iteration = 6
+batch_size = 6
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+few_shot_score_threshold = 0.8
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/configs/ifbench_gepa_local.toml

@@ -0,0 +1,57 @@
+# GEPA Prompt Learning for IFBench
+# Local backend configuration targeting the IFBench task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8111"
+task_app_id = "ifbench"
+
+# Candidate evaluation seeds during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out pool used for the final comparison sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "ifbench_following"
+name = "IFBench Instruction Following"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an obedient assistant that must follow instructions exactly. Ensure that every requirement is satisfied, avoid unsolicited commentary, and be explicit when information is missing."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Instruction: {instruction}\n\nInput: {input}\n\nProvide the response that best follows the instruction."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+instruction = "REQUIRED"
+input = "OPTIONAL"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "ifbench-gepa"
+
+[prompt_learning.gepa]
+env_name = "ifbench"
+initial_population_size = 24
+num_generations = 12
+mutation_rate = 0.3
+crossover_rate = 0.6
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.5
+children_per_generation = 12
+patience_generations = 4
+rollout_budget = 480
+archive_size = 32
+pareto_eps = 1e-6
+max_concurrent_rollouts = 20
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"

examples/blog_posts/gepa/configs/ifbench_gepa_qwen.toml

@@ -0,0 +1,35 @@
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-web-web.modal.run" # TODO: replace with HotpotQA task app URL
+task_app_id = "hotpotqa"
+
+# Seeds
+evaluation_seeds = [0,1,2,3,4,5,6,7,8,9]
+
+# Held-out validation
+validation_seeds = [10,11,12,13,14,15,16,17,18,19]
+validation_pool = "validation"
+validation_top_k = 3
+
+# Train split configuration
+[prompt_learning.env_config]
+pool = "train"
+
+# Policy model (synth Qwen via backend inference proxy)
+[prompt_learning.policy]
+provider = "synth"
+model = "Qwen/Qwen3-8B"
+
+# GEPA parameters (tune as needed)
+[prompt_learning.gepa]
+env_name = "ifbench"
+initial_population_size = 24
+num_generations = 6
+children_per_generation = 12
+minibatch_size = 10
+pareto_set_size = 32
+rollout_budget = 600
+max_concurrent_rollouts = 16
+mutation_llm_model = "openai/gpt-oss-120b"
+mutation_llm_provider = "groq"
+proposer_type = "dspy"

examples/blog_posts/gepa/configs/ifbench_mipro_local.toml

@@ -0,0 +1,51 @@
+# MIPROv2 Prompt Learning for IFBench
+# Local backend configuration targeting the IFBench task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8111"
+task_app_id = "ifbench"
+
+# Seeds evaluated during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final comparison
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "ifbench_following"
+name = "IFBench Instruction Following"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an obedient assistant that follows instructions exactly. Ensure that every constraint is satisfied and mention explicitly if something cannot be completed."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Instruction: {instruction}\n\nInput: {input}\n\nReturn the response that best follows the instruction."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+instruction = "REQUIRED"
+input = "OPTIONAL"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "ifbench-mipro"
+
+[prompt_learning.mipro]
+env_name = "ifbench"
+num_iterations = 16
+num_evaluations_per_iteration = 6
+batch_size = 6
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+few_shot_score_threshold = 0.8
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]

examples/blog_posts/gepa/configs/pupa_gepa_local.toml

@@ -0,0 +1,58 @@
+# GEPA Prompt Learning for PUPA
+# Local backend configuration targeting the PUPA privacy-aware delegation task app.
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "http://127.0.0.1:8113"
+task_app_id = "pupa"
+
+# Seeds explored during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final evaluation
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "pupa_privacy"
+name = "PUPA Privacy-Constrained Delegation"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an assistant that must complete the task while honouring every privacy rule. Never reveal disallowed fields, always justify decisions, and explicitly state when data cannot be shared."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Task: {task}\n\nPrivacy Policy: {policy}\n\nRecords:\n{records}\n\nProduce the delegated plan or response while respecting the privacy policy."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+task = "REQUIRED"
+policy = "REQUIRED"
+records = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "pupa-gepa"
+
+[prompt_learning.gepa]
+env_name = "pupa"
+initial_population_size = 24
+num_generations = 15
+mutation_rate = 0.3
+crossover_rate = 0.6
+selection_pressure = 1.0
+minibatch_size = 8
+pareto_set_size = 24
+feedback_fraction = 0.6
+children_per_generation = 12
+patience_generations = 5
+rollout_budget = 540
+archive_size = 36
+pareto_eps = 1e-6
+max_concurrent_rollouts = 20
+mutation_llm_model = "openai/gpt-oss-20b"
+mutation_llm_provider = "groq"

examples/blog_posts/gepa/configs/pupa_mipro_local.toml

@@ -0,0 +1,52 @@
+# MIPROv2 Prompt Learning for PUPA
+# Local backend configuration targeting the PUPA privacy-aware delegation task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8113"
+task_app_id = "pupa"
+
+# Seeds evaluated during optimisation
+evaluation_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+# Held-out seeds for the final sweep
+test_pool = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[prompt_learning.initial_prompt]
+id = "pupa_privacy"
+name = "PUPA Privacy-Constrained Delegation"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an assistant that must complete tasks without violating the privacy policy. Redact any forbidden attributes and justify refusals."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Task: {task}\n\nPrivacy Policy: {policy}\n\nRecords:\n{records}\n\nProvide the delegated plan or answer, ensuring compliance with the privacy policy."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+task = "REQUIRED"
+policy = "REQUIRED"
+records = "REQUIRED"
+
+[prompt_learning.policy]
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 512
+policy_name = "pupa-mipro"
+
+[prompt_learning.mipro]
+env_name = "pupa"
+num_iterations = 20
+num_evaluations_per_iteration = 6
+batch_size = 6
+max_concurrent = 16
+meta_model = "gpt-4.1-mini"
+meta_model_provider = "openai"
+few_shot_score_threshold = 0.85
+test_pool = [20, 21, 22, 23, 24]
+bootstrap_train_seeds = [0, 1, 2, 3, 4]
+online_pool = [5, 6, 7, 8, 9]