lm-deluge 0.0.67__py3-none-any.whl → 0.0.90__py3-none-any.whl

lm_deluge/pipelines/gepa/examples/01_synthetic_keywords.py

@@ -0,0 +1,140 @@
+"""
+Example 1: Synthetic Keyword Matching Task
+
+This is the simplest possible GEPA example - no LLM calls needed for the task itself.
+The goal is to evolve a prompt that contains certain target keywords.
+
+This example is useful for:
+- Understanding GEPA's basic mechanics
+- Testing without API costs
+- Debugging your setup
+
+Run:
+    python 01_synthetic_keywords.py
+"""
+
+from lm_deluge import LLMClient
+from lm_deluge.pipelines.gepa import Component, EvalResult, GEPAEngine
+from lm_deluge.prompt import Conversation, Message
+
+
+def main():
+    # Target: We want a prompt that mentions these concepts
+    TARGET_KEYWORDS = {
+        "step",
+        "by",
+        "think",
+        "carefully",
+        "show",
+        "work",
+        "verify",
+        "answer",
+    }
+
+    def evaluate(
+        client: LLMClient,  # type: ignore
+        component_values: dict[str, str],
+        example: dict,
+    ) -> EvalResult:
+        """Score based on keyword coverage - no LLM needed."""
+        prompt = component_values["system_prompt"]
+        words = set(prompt.lower().split())
+        matches = len(words & TARGET_KEYWORDS)
+        score = matches / len(TARGET_KEYWORDS)
+
+        # Build a minimal conversation (required by EvalResult)
+        conv = Conversation.system(prompt)
+        conv = conv.add(Message.user(example["question"]))
+        conv = conv.add(
+            Message.ai(f"[Keyword score: {matches}/{len(TARGET_KEYWORDS)}]")
+        )
+
+        feedback = f"""Score: {score:.2f}
+Keywords found: {words & TARGET_KEYWORDS}
+Keywords missing: {TARGET_KEYWORDS - words}"""
+
+        return EvalResult(conversation=conv, score=score, feedback=feedback)
+
+    # Mock proposer that simulates what an LLM might suggest
+    # In real usage, this would be an actual LLM
+    iteration = [0]
+    improvements = [
+        "Think step by step before answering.",
+        "Think carefully, step by step. Show your work.",
+        "Think carefully, step by step. Show your work and verify your answer.",
+        "Think carefully and reason step by step. Show your work, then verify your answer is correct.",
+    ]
+
+    class MockProposerClient:
+        """Fake LLMClient that returns predetermined improvements."""
+
+        def process_prompts_sync(self, prompts, **kwargs):
+            iteration[0] += 1
+            idx = min(iteration[0], len(improvements) - 1)
+
+            class FakeResponse:
+                completion = f"""COMPONENT: system_prompt
+REASONING: Adding more target keywords to improve coverage.
+NEW_VALUE:
+```
+{improvements[idx]}
+```"""
+
+            return [FakeResponse()]
+
+    # Simple dataset (content doesn't matter for this toy task)
+    dataset = [{"question": f"Question {i}"} for i in range(10)]
+
+    print("=" * 60)
+    print("GEPA Example 1: Synthetic Keyword Matching")
+    print("=" * 60)
+    print(f"Target keywords: {TARGET_KEYWORDS}")
+    print()
+
+    # Define component to optimize
+    components = {
+        "system_prompt": Component(
+            description="System prompt to optimize for keyword coverage",
+            value="You are a helpful assistant.",
+        ),
+    }
+
+    # Create engine with mock clients
+    engine = GEPAEngine(
+        components=components,
+        evaluate_fn=evaluate,  # type: ignore[arg-type]
+        dataset=dataset[:7],
+        val_dataset=dataset[7:],
+        task_client=MockProposerClient(),  # type: ignore[arg-type]
+        proposer_client=MockProposerClient(),  # type: ignore[arg-type]
+        max_iterations=10,
+        max_evals=100,
+        minibatch_size=2,
+        seed=42,
+    )
+
+    # Run optimization
+    result = engine.run()  # type: ignore[func-returns-value]
+
+    print()
+    print("=" * 60)
+    print("Results")
+    print("=" * 60)
+    print(f"Candidates discovered: {result.num_candidates}")  # type: ignore[union-attr]
+    print(f"Best score: {result.best_score:.2%}")  # type: ignore[union-attr]
+    print(f"Total evaluations: {result.total_evals}")  # type: ignore[union-attr]
+    print()
+    print("Best prompt found:")
+    print("-" * 40)
+    print(result.best_candidate["system_prompt"])  # type: ignore[union-attr]
+    print("-" * 40)
+
+    # Show evolution
+    print()
+    print("Evolution of candidates:")
+    for i, (idx, candidate, score) in enumerate(result.best_k(5)):  # type: ignore[union-attr]
+        print(f"  {i+1}. Score={score:.2%}: {candidate['system_prompt'][:60]}...")
+
+
+if __name__ == "__main__":
+    main()

lm_deluge/pipelines/gepa/examples/02_gsm8k_math.py

@@ -0,0 +1,261 @@
+"""
+Example: GSM8K Math Reasoning
+
+Optimize a system prompt for grade school math problems.
+This is a classic prompt optimization task - improve accuracy on GSM8K.
+
+The task:
+- Input: Math word problem
+- Output: Numerical answer
+- Score: Exact match with ground truth
+
+Run:
+    python 02_gsm8k_math.py
+
+Requirements:
+    pip install datasets
+    # Set OPENAI_API_KEY or ANTHROPIC_API_KEY environment variable
+"""
+
+import os
+import re
+import sys
+from typing import cast
+
+import dotenv
+
+from lm_deluge.client import LLMClient, _LLMClient
+from lm_deluge.pipelines.gepa import Component, EvalResult, optimize
+from lm_deluge.prompt import Conversation, Message
+
+dotenv.load_dotenv()
+
+
+def load_gsm8k_sample(
+    n_train: int = 50, n_val: int = 20
+) -> tuple[list[dict], list[dict]]:
+    """Load a sample of GSM8K problems."""
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        print("Please install datasets: pip install datasets")
+        sys.exit(1)
+
+    print("Loading GSM8K dataset...")
+    ds = load_dataset("openai/gsm8k", "main", split="train")
+
+    # Extract answer from "#### <number>" format
+    def extract_answer(answer_text: str) -> str:
+        match = re.search(r"####\s*(-?\d[\d,]*\.?\d*)", answer_text)
+        if match:
+            return match.group(1).replace(",", "")
+        return answer_text.strip()
+
+    data = []
+    for item in ds:
+        item = cast(dict[str, str], item)
+        data.append(
+            {
+                "question": item["question"],
+                "answer": extract_answer(item["answer"]),
+            }
+        )
+
+    # Shuffle and split
+    import random
+
+    random.seed(42)
+    random.shuffle(data)
+
+    return data[:n_train], data[n_train : n_train + n_val]
+
+
+def extract_final_number(text: str) -> str | None:
+    """Extract the final number from model output."""
+    # Look for common patterns
+    patterns = [
+        r"(?:answer|result|total|=)\s*[:=]?\s*\$?(-?\d[\d,]*\.?\d*)",
+        r"####\s*(-?\d[\d,]*\.?\d*)",
+        r"\*\*(-?\d[\d,]*\.?\d*)\*\*",
+        r"(-?\d[\d,]*\.?\d*)\s*$",  # Last number in text
+    ]
+
+    for pattern in patterns:
+        matches = re.findall(pattern, text, re.IGNORECASE)
+        if matches:
+            return matches[-1].replace(",", "")
+
+    # Fallback: find all numbers and return the last one
+    numbers = re.findall(r"-?\d[\d,]*\.?\d*", text)
+    if numbers:
+        return numbers[-1].replace(",", "")
+
+    return None
+
+
+async def evaluate(
+    client: _LLMClient, component_values: dict[str, str], example: dict
+) -> EvalResult:
+    """Evaluate one math problem."""
+    # Build conversation
+    conv = Conversation.system(component_values["system_prompt"])
+    user_msg = f"""Problem: {example["question"]}
+
+Solve this step by step, then provide your final numerical answer."""
+    conv = conv.add(Message.user(user_msg))
+
+    # Run inference (async)
+    response = await client.start(conv)
+    output = response.completion or ""
+
+    # Extract and score
+    predicted = extract_final_number(output)
+    expected = example["answer"]
+
+    if predicted is None:
+        score = 0.0
+    else:
+        try:
+            pred_num = float(predicted)
+            exp_num = float(expected)
+            score = 1.0 if abs(pred_num - exp_num) < 0.01 else 0.0
+        except ValueError:
+            score = 1.0 if predicted.strip() == expected.strip() else 0.0
+
+    # Build feedback for the proposer
+    if score == 1.0:
+        feedback = f"""Score: 1.0 (CORRECT)
+Question: {example["question"][:100]}...
+Expected: {expected}
+Got: {predicted}"""
+    else:
+        feedback = f"""Score: 0.0 (INCORRECT)
+Question: {example["question"]}
+Expected answer: {expected}
+Model extracted answer: {predicted}
+Model reasoning: {output[:500]}{"..." if len(output) > 500 else ""}
+
+The model either made a calculation error or failed to extract the answer properly."""
+
+    # Return full trajectory
+    full_conv = conv.add(Message.ai(output))
+    return EvalResult(conversation=full_conv, score=score, feedback=feedback)
+
+
+def main():
+    # Check for API keys
+    model = None
+    proposer_model = None
+
+    if os.getenv("OPENAI_API_KEY"):
+        model = "gpt-4.1-nano"
+        proposer_model = "gpt-5-mini"
+    elif os.getenv("ANTHROPIC_API_KEY"):
+        model = "claude-3-5-haiku-latest"
+        proposer_model = "claude-sonnet-4-20250514"
+    else:
+        print("Please set OPENAI_API_KEY or ANTHROPIC_API_KEY")
+        sys.exit(1)
+
+    print(f"Using task model: {model}")
+    print(f"Using proposer model: {proposer_model}")
+
+    # Load data - larger val set for more room to improve
+    trainset, valset = load_gsm8k_sample(n_train=50, n_val=50)
+    print(f"Loaded {len(trainset)} training, {len(valset)} validation examples")
+
+    # Create clients
+    task_client = LLMClient(
+        model,
+        max_requests_per_minute=100,
+        max_new_tokens=512,
+        temperature=0.0,
+    )
+    proposer_client = LLMClient(
+        proposer_model,
+        max_requests_per_minute=50,
+        max_new_tokens=1024,
+    )
+
+    # Define component to optimize
+    components = {
+        "system_prompt": Component(
+            description="System prompt that instructs the model how to solve math problems step by step",
+            value="You are a helpful math tutor. Solve math problems step by step.",
+        ),
+    }
+
+    print()
+    print("=" * 60)
+    print("GEPA Example: GSM8K Math Reasoning")
+    print("=" * 60)
+    print(f"Seed prompt: {components['system_prompt'].value}")
+    print()
+
+    # Meta-instructions to guide the proposer
+    meta_instructions = """
+- Focus on GENERAL improvements that will help across many problems, not just the specific example shown
+- Don't add problem-specific details (like "mosquito infections" or "baguette rates") to the prompt
+- Keep the prompt concise - longer is not always better
+- Prioritize clarity and unambiguous instructions over covering edge cases
+""".strip()
+
+    # Run optimization
+    result = optimize(
+        components=components,
+        evaluate_fn=evaluate,
+        dataset=trainset,
+        val_dataset=valset,
+        task_client=task_client,
+        proposer_client=proposer_client,
+        max_iterations=30,
+        max_evals=500,
+        minibatch_size=5,
+        meta_instructions=meta_instructions,
+        run_dir="./gsm8k_gepa",
+        save_trajectories=True,
+        seed=42,
+    )
+
+    print()
+    print("=" * 60)
+    print("Results")
+    print("=" * 60)
+    print(f"Candidates discovered: {result.num_candidates}")
+    print(f"Best validation accuracy: {result.best_score:.1%}")
+    print(f"Total evaluations: {result.total_evals}")
+    print()
+    print("Best prompt found:")
+    print("-" * 40)
+    print(result.best_candidate["system_prompt"])
+    print("-" * 40)
+
+    # Show top candidates
+    print()
+    print("Top 3 candidates:")
+    for i, (idx, candidate, score) in enumerate(result.best_k(3)):
+        print(f"\n{i + 1}. Score={score:.1%}")
+        print(f"   {candidate['system_prompt'][:100]}...")
+
+    # Show lineage of best
+    print()
+    print(f"Lineage of best candidate (idx={result.best_idx}):")
+    lineage = result.lineage(result.best_idx)
+    for i, idx in enumerate(lineage):
+        score = result.candidate_avg_scores[idx]
+        prompt_preview = result.candidates[idx]["system_prompt"][:50]
+        print(f"  {'→ ' if i > 0 else ''}{idx}: {score:.1%} - {prompt_preview}...")
+
+    # Print cost summary
+    print()
+    print("=" * 60)
+    print("Cost Summary")
+    print("=" * 60)
+    print("Task client (evaluations):")
+    task_client.print_usage()
+    print("\nProposer client (proposals):")
+    proposer_client.print_usage()
+
+
+if __name__ == "__main__":
+    main()

lm_deluge/pipelines/gepa/examples/03_hotpotqa_multihop.py

@@ -0,0 +1,300 @@
+"""
+Example: HotpotQA Multi-hop Question Answering
+
+Optimize a system prompt for multi-hop reasoning questions.
+This task requires combining information from multiple sources.
+
+The task:
+- Input: Question + supporting context paragraphs
+- Output: Short answer
+- Score: F1 overlap with ground truth
+
+This example demonstrates:
+- Multi-component optimization (system_prompt + answer_format)
+- More complex scoring (F1 instead of exact match)
+- Richer trajectory information for reflection
+
+Run:
+    python 03_hotpotqa_multihop.py
+
+Requirements:
+    pip install datasets
+    # Set OPENAI_API_KEY or ANTHROPIC_API_KEY environment variable
+"""
+
+import os
+import re
+import string
+import sys
+from collections import Counter
+
+import dotenv
+
+from lm_deluge import LLMClient
+from lm_deluge.pipelines.gepa import Component, EvalResult, optimize
+from lm_deluge.prompt import Conversation, Message
+
+dotenv.load_dotenv()
+
+
+def load_hotpotqa_sample(
+    n_train: int = 40, n_val: int = 20
+) -> tuple[list[dict], list[dict]]:
+    """Load a sample of HotpotQA problems."""
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        print("Please install datasets: pip install datasets")
+        sys.exit(1)
+
+    print("Loading HotpotQA dataset...")
+    ds = load_dataset(
+        "hotpot_qa", "distractor", split="validation", trust_remote_code=True
+    )
+
+    data = []
+    for item in ds:  # type: ignore
+        # Combine supporting facts into context
+        context_parts = []
+        for title, sentences in zip(
+            item["context"]["title"],  # type: ignore
+            item["context"]["sentences"],  # type: ignore
+        ):
+            context_parts.append(f"[{title}]\n" + " ".join(sentences))
+
+        data.append(
+            {
+                "question": item["question"],  # type: ignore
+                "context": "\n\n".join(context_parts),
+                "answer": item["answer"],  # type: ignore
+                "type": item["type"],  # type: ignore  # 'comparison' or 'bridge'
+            }
+        )
+
+    # Shuffle and split
+    import random
+
+    random.seed(42)
+    random.shuffle(data)
+
+    return data[:n_train], data[n_train : n_train + n_val]
+
+
+def normalize_answer(s: str) -> str:
+    """Normalize answer for comparison."""
+
+    def remove_articles(text):
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+
+    def white_space_fix(text):
+        return " ".join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def compute_f1(prediction: str, ground_truth: str) -> float:
+    """Compute F1 score between prediction and ground truth."""
+    pred_tokens = normalize_answer(prediction).split()
+    gt_tokens = normalize_answer(ground_truth).split()
+
+    if not pred_tokens or not gt_tokens:
+        return float(pred_tokens == gt_tokens)
+
+    common = Counter(pred_tokens) & Counter(gt_tokens)
+    num_same = sum(common.values())
+
+    if num_same == 0:
+        return 0.0
+
+    precision = num_same / len(pred_tokens)
+    recall = num_same / len(gt_tokens)
+    f1 = (2 * precision * recall) / (precision + recall)
+
+    return f1
+
+
+def extract_answer(output: str) -> str:
+    """Extract answer from model output."""
+    if "Answer:" in output:
+        return output.split("Answer:")[-1].strip()
+    elif "answer:" in output.lower():
+        return output.lower().split("answer:")[-1].strip()
+    else:
+        # Take first sentence/line
+        return output.split("\n")[0].split(".")[0].strip()
+
+
+def make_evaluate_fn(task_client: LLMClient):  # type: ignore
+    """Create the evaluate function."""
+
+    def evaluate(
+        client: LLMClient,  # type: ignore
+        component_values: dict[str, str],
+        example: dict,
+    ) -> EvalResult:
+        """Evaluate one HotpotQA question."""
+        # Build conversation
+        conv = Conversation.system(component_values["system_prompt"])
+
+        user_msg = f"""Context:
+{example['context']}
+
+Question: {example['question']}
+
+{component_values['answer_format']}"""
+        conv = conv.add(Message.user(user_msg))
+
+        # Run inference
+        response = client.process_prompts_sync([conv], show_progress=False)[0]
+        output = response.completion or ""
+
+        # Extract answer and compute F1
+        extracted = extract_answer(output)
+        f1 = compute_f1(extracted, example["answer"])
+
+        # Build detailed feedback
+        if f1 >= 0.8:
+            feedback = f"""Score: {f1:.2f} (GOOD)
+Question type: {example['type']}
+Expected: {example['answer']}
+Got: {extracted}"""
+        else:
+            hint = ""
+            if example["type"] == "comparison":
+                hint = "This requires comparing two entities."
+            else:
+                hint = "This requires following a chain of reasoning."
+
+            feedback = f"""Score: {f1:.2f} (NEEDS IMPROVEMENT)
+Question type: {example['type']}
+Expected: {example['answer']}
+Got: {extracted}
+Model output: {output[:300]}{'...' if len(output) > 300 else ''}
+Hint: {hint}"""
+
+        # Return full trajectory
+        full_conv = conv.add(Message.ai(output))
+        return EvalResult(conversation=full_conv, score=f1, feedback=feedback)
+
+    return evaluate
+
+
+def main():
+    # Check for API keys
+    model = None
+    proposer_model = None
+
+    if os.getenv("OPENAI_API_KEY"):
+        model = "gpt-4.1-nano"
+        proposer_model = "gpt-4.1-mini"
+    elif os.getenv("ANTHROPIC_API_KEY"):
+        model = "claude-3-5-haiku-latest"
+        proposer_model = "claude-sonnet-4-20250514"
+    else:
+        print("Please set OPENAI_API_KEY or ANTHROPIC_API_KEY")
+        sys.exit(1)
+
+    print(f"Using task model: {model}")
+    print(f"Using proposer model: {proposer_model}")
+
+    # Load data
+    trainset, valset = load_hotpotqa_sample(n_train=30, n_val=15)
+    print(f"Loaded {len(trainset)} training, {len(valset)} validation examples")
+
+    # Show question type distribution
+    train_types = Counter(x["type"] for x in trainset)
+    print(f"Training set types: {dict(train_types)}")
+
+    # Create clients
+    task_client = LLMClient(  # type: ignore[operator]
+        model,
+        max_requests_per_minute=100,
+        max_new_tokens=256,
+        temperature=0.0,
+    )
+    proposer_client = LLMClient(  # type: ignore[operator]
+        proposer_model,
+        max_requests_per_minute=50,
+        max_new_tokens=1024,
+    )
+
+    # Define components to optimize (two components this time)
+    components = {
+        "system_prompt": Component(
+            description="System prompt that guides the model's reasoning approach",
+            value="You are a helpful assistant that answers questions based on the provided context.",
+        ),
+        "answer_format": Component(
+            description="Instructions for how the model should format its answer",
+            value="Provide a short, direct answer to the question.",
+        ),
+    }
+
+    print()
+    print("=" * 60)
+    print("GEPA Example: HotpotQA Multi-hop QA")
+    print("=" * 60)
+    print("Components being optimized:")
+    for name, comp in components.items():
+        print(f"  - {name}: {comp.value[:50]}...")
+    print()
+
+    # Run optimization
+    result = optimize(
+        components=components,
+        evaluate_fn=make_evaluate_fn(task_client),  # type: ignore[arg-type]
+        dataset=trainset,
+        val_dataset=valset,
+        task_client=task_client,
+        proposer_client=proposer_client,
+        max_iterations=20,
+        max_evals=250,
+        minibatch_size=3,
+        run_dir="./hotpotqa_gepa",
+        save_trajectories=True,
+        seed=42,
+    )
+
+    print()
+    print("=" * 60)
+    print("Results")
+    print("=" * 60)
+    print(f"Candidates discovered: {result.num_candidates}")
+    print(f"Best validation F1: {result.best_score:.1%}")
+    print(f"Total evaluations: {result.total_evals}")
+    print()
+    print("Best candidate found:")
+    print("-" * 40)
+    for name, text in result.best_candidate.items():
+        print(f"{name}:")
+        print(f"  {text}")
+        print()
+    print("-" * 40)
+
+    # Show improvement
+    seed_score = result.candidate_avg_scores[0]
+    improvement = result.best_score - seed_score
+    print(
+        f"\nImprovement over seed: {seed_score:.1%} → {result.best_score:.1%} (+{improvement:.1%})"
+    )
+
+    # Show which component changed most
+    if result.num_candidates > 1:
+        diff = result.diff(0, result.best_idx)
+        print("\nChanges from seed to best:")
+        for comp, (old, new) in diff.items():
+            if old != new:
+                print(f"  {comp}:")
+                print(f"    OLD: {old[:60]}...")
+                print(f"    NEW: {new[:60]}...")
+
+
+if __name__ == "__main__":
+    main()