pi-para 0.6.0 → 0.6.2

package/scripts/gepa/metric.py

@@ -1,279 +0,0 @@
-"""LLM-as-judge metric for GEPA prompt optimization.
-
-Returns dspy.Prediction(score=float, feedback=str) — GEPA reads the
-textual feedback to guide instruction evolution.
-
-One LLM judge call per (candidate, example) evaluation.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-
-import dspy
-
-logger = logging.getLogger(__name__)
-
-# -- Judge prompt template ----------------------------------------------------
-
-JUDGE_SYSTEM = """You are an expert evaluator for PARA wiki knowledge management prompts and instructions.
-
-You score how well a wiki system's output follows the PARA methodology:
-- **Projects/**: Goals with end dates
-- **Areas/**: Ongoing responsibilities
-- **Resources/**: Reference docs, how-tos, patterns (default for most pages)
-- **Archives/**: Completed items
-
-Good wiki output has: structured sections (Topic, Key Facts, Connections), 
-[[wikilinks]] to related pages, kebab-case scope/tags, no exposed secrets, 
-and clear actionable content."""
-
-JUDGE_PROMPT = """Evaluate the following wiki output produced by an instruction/prompt.
-
-TARGET TYPE: {target_type}
-TARGET PURPOSE: {target_purpose}
-
-TASK CONTEXT (input given to the instruction):
-{task_context}
-
-PRODUCED OUTPUT (generated using the instruction):
-{produced_output}
-
-REFERENCE OUTPUT (real wiki page for comparison):
-{expected_output}
-
-Score on these 6 dimensions (each 0.0 to 1.0):
-
-1. **Structure**: Has proper sections (Topic, Key Facts, Connections). Uses markdown headers. Reasonable length (200-5000 chars for body). Not just a wall of text.
-2. **PARA Compliance**: Would produce correct category assignment. Scope mentions are kebab-case project names (not topic descriptions). Tags would be kebab-case.
-3. **Cross-references**: Mentions or produces [[wikilinks]]. References related concepts. Would include a Connections section.
-4. **Security**: Does not expose API keys, tokens, passwords. Documents WHERE secrets are stored, not values.
-5. **Completeness**: Covers the topic adequately. Includes key facts. Doesn't miss obvious important aspects.
-6. **Actionability**: Output is clear and specific. Follows concrete rules. Not vague or ambiguous.
-
-Respond ONLY with valid JSON (no markdown fences):
-{{"dimensions": [{{"name": "structure", "score": 0.0, "feedback": "..."}}, {{"name": "para_compliance", "score": 0.0, "feedback": "..."}}, {{"name": "cross_references", "score": 0.0, "feedback": "..."}}, {{"name": "security", "score": 0.0, "feedback": "..."}}, {{"name": "completeness", "score": 0.0, "feedback": "..."}}, {{"name": "actionability", "score": 0.0, "feedback": "..."}}], "overall_feedback": "..."}}"""
-
-
-# -- Dimension weights --------------------------------------------------------
-
-DIMENSION_WEIGHTS = {
-    "structure": 0.15,
-    "para_compliance": 0.20,
-    "cross_references": 0.15,
-    "security": 0.15,
-    "completeness": 0.15,
-    "actionability": 0.20,
-}
-
-# -- Target purposes ----------------------------------------------------------
-
-TARGET_PURPOSES = {
-    "capture": "Guide an LLM to extract knowledge from coding sessions into PARA wiki pages",
-    "ingest": "Guide an LLM to organize ingested source material into structured wiki pages",
-    "query": "Guide an LLM to synthesize answers from wiki search results with citations",
-    "maintenance": "Guide an LLM to maintain wiki quality: dedup, links, categories, staleness",
-    "tool": "Describe a wiki tool's purpose so the LLM knows when and how to use it",
-    "skill": "Behavioral guidelines for actively using the PARA wiki during work",
-    "system": "System prompt for a standalone wiki knowledge management agent",
-}
-
-
-# -- JSON parsing helpers -----------------------------------------------------
-
-def _parse_judge_response(text: str) -> dict | None:
-    """Parse JSON from LLM judge response, handling markdown fences."""
-    # Strip markdown code fences
-    text = text.strip()
-    if text.startswith("```"):
-        # Remove first and last lines (fences)
-        lines = text.split("\n")
-        text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
-
-    # Try direct parse
-    try:
-        return json.loads(text)
-    except json.JSONDecodeError:
-        pass
-
-    # Try finding JSON in the text
-    match = re.search(r'\{.*\}', text, re.DOTALL)
-    if match:
-        try:
-            return json.loads(match.group())
-        except json.JSONDecodeError:
-            pass
-
-    return None
-
-
-def _compute_score(parsed: dict) -> tuple[float, str]:
-    """Compute weighted score and formatted feedback from parsed judge response."""
-    dimensions = parsed.get("dimensions", [])
-    if not dimensions:
-        return 0.0, "No dimensions in judge response"
-
-    total_score = 0.0
-    total_weight = 0.0
-    feedback_parts = []
-
-    for dim in dimensions:
-        name = dim.get("name", "unknown")
-        score = float(dim.get("score", 0.0))
-        fb = dim.get("feedback", "")
-        weight = DIMENSION_WEIGHTS.get(name, 1.0 / len(dimensions))
-
-        total_score += score * weight
-        total_weight += weight
-        feedback_parts.append(f"{name}={score:.1f}: {fb}")
-
-    overall = parsed.get("overall_feedback", "")
-    if overall:
-        feedback_parts.append(f"Overall: {overall}")
-
-    final_score = total_score / total_weight if total_weight > 0 else 0.0
-    feedback = " | ".join(feedback_parts)
-
-    return final_score, feedback[:500]  # Cap feedback length
-
-
-# -- Judge LM holder ----------------------------------------------------------
-
-_judge_lm: dspy.LM | None = None
-
-
-def set_judge_lm(lm) -> None:
-    """Set the LLM used for judging. Must be called before using the metric."""
-    global _judge_lm
-    _judge_lm = lm
-
-
-# -- Main metric function -----------------------------------------------------
-
-def wiki_quality_metric(
-    gold: dspy.Example,
-    pred: dspy.Prediction | None,
-    trace=None,
-    pred_name: str | None = None,
-    pred_trace=None,
-) -> dspy.Prediction:
-    """LLM-as-judge metric for wiki output quality.
-
-    Scores on 6 dimensions with rich textual feedback.
-    Returns dspy.Prediction(score=float, feedback=str).
-    """
-    # Handle failed prediction
-    if pred is None:
-        return dspy.Prediction(
-            score=0.0,
-            feedback="Prediction failed — no output generated."
-        )
-
-    produced_output = getattr(pred, "output", "") or ""
-    if not produced_output.strip():
-        return dspy.Prediction(
-            score=0.0,
-            feedback="Empty output — the instruction produced nothing."
-        )
-
-    task_context = getattr(gold, "task_context", "")
-    expected_output = getattr(gold, "expected_output", "")
-
-    # Determine target type from metadata or default
-    target_type = getattr(gold, "target_type", "capture")
-    target_purpose = TARGET_PURPOSES.get(target_type, TARGET_PURPOSES["capture"])
-
-    # Truncate for judge context window
-    produced_trunc = produced_output[:2000]
-    expected_trunc = expected_output[:2000]
-    context_trunc = task_context[:500]
-
-    # Build judge prompt
-    judge_prompt = JUDGE_PROMPT.format(
-        target_type=target_type,
-        target_purpose=target_purpose,
-        task_context=context_trunc,
-        produced_output=produced_trunc,
-        expected_output=expected_trunc,
-    )
-
-    # Call judge LLM
-    judge = _judge_lm or dspy.settings.lm
-    if judge is None:
-        # Fallback: basic heuristic if no judge LM available
-        return _heuristic_fallback(produced_output, expected_output)
-
-    try:
-        response = judge(
-            messages=[
-                {"role": "system", "content": JUDGE_SYSTEM},
-                {"role": "user", "content": judge_prompt},
-            ],
-            temperature=0.0,
-            max_tokens=1000,
-        )
-        response_text = response[0].get("text", "") if response else ""
-    except Exception as e:
-        logger.warning("Judge LLM call failed: %s", e)
-        return _heuristic_fallback(produced_output, expected_output)
-
-    # Parse judge response
-    parsed = _parse_judge_response(response_text)
-    if not parsed:
-        logger.warning("Failed to parse judge response: %s", response_text[:200])
-        return _heuristic_fallback(produced_output, expected_output)
-
-    score, feedback = _compute_score(parsed)
-
-    return dspy.Prediction(score=score, feedback=feedback)
-
-
-def _heuristic_fallback(output: str, expected: str) -> dspy.Prediction:
-    """Simple heuristic scoring when judge LLM is unavailable."""
-    score = 0.0
-    parts = []
-
-    # Structure check
-    has_sections = bool(re.search(r"^## ", output, re.MULTILINE))
-    if has_sections:
-        score += 0.15
-    else:
-        parts.append("structure=0: no ## sections")
-
-    # Wikilinks check
-    has_wikilinks = "[[" in output and "]]" in output
-    if has_wikilinks:
-        score += 0.15
-    else:
-        parts.append("cross_refs=0: no [[wikilinks]]")
-
-    # Length check
-    if 200 <= len(output) <= 5000:
-        score += 0.15
-    else:
-        parts.append(f"length issue: {len(output)} chars")
-
-    # Security — no obvious secrets
-    has_secrets = bool(re.search(r"sk-[a-zA-Z0-9]{20,}|api[_-]?key\s*[:=]\s*['\"][^'\"]{10,}", output, re.IGNORECASE))
-    if not has_secrets:
-        score += 0.15
-    else:
-        parts.append("security=0: possible secret detected")
-
-    # Content overlap with expected
-    expected_words = set(expected.lower().split())
-    output_words = set(output.lower().split())
-    if expected_words:
-        overlap = len(expected_words & output_words) / len(expected_words)
-        score += overlap * 0.2
-    else:
-        score += 0.1
-
-    # Non-empty
-    if len(output) > 100:
-        score += 0.1
-
-    feedback = " | ".join(parts) if parts else "Heuristic: acceptable output"
-    return dspy.Prediction(score=min(score, 1.0), feedback=f"[heuristic] {feedback}")

package/scripts/gepa/optimize.py

@@ -1,367 +0,0 @@
-#!/usr/bin/env python3
-"""GEPA optimizer for pi-para prompts, tools, and skills.
-
-Usage (via uv):
-    uv run --project scripts/gepa scripts/gepa/optimize.py \
-        --targets-file ~/.pi/wiki/gepa/input/targets.json \
-        --wiki-dir ~/.pi/wiki \
-        --output ~/.pi/wiki/gepa/output/results.json \
-        --model anthropic/claude-sonnet-4-20250514 \
-        --auto light
-
-The TypeScript orchestrator prepares targets.json and reads results.json.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import os
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-
-import dspy
-
-# Local imports (same directory)
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from lm_providers import create_lm
-from program import PromptProxy, extract_evolved_instruction
-from metric import wiki_quality_metric, set_judge_lm
-from dataset import build_dataset, get_wiki_context
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="[gepa] %(asctime)s %(levelname)s %(message)s",
-    datefmt="%H:%M:%S",
-)
-logger = logging.getLogger(__name__)
-
-
-# -- Target type mapping -------------------------------------------------------
-
-TARGET_TYPE_MAP = {
-    # Prompt templates
-    "wiki-system-prompt": "system",
-    "ingest-prompt": "ingest",
-    "query-prompt": "query",
-    "capture-system-prompt": "capture",
-    "capture-prompt": "capture",
-    "explicit-capture-prompt": "capture",
-    "summarize-system-prompt": "system",
-    "iterative-update-prompt": "ingest",
-    "overview-prompt": "query",
-    "lint-prompt": "maintenance",
-    "maintenance-system-prompt": "maintenance",
-    "processor-capture-prompt": "capture",
-    # Tool instructions
-    "tool-wiki-ingest": "tool",
-    "tool-wiki-query": "tool",
-    "tool-wiki-write": "tool",
-    "tool-wiki-read": "tool",
-    "tool-wiki-summarize": "tool",
-    # Skill instructions
-    "skill-para": "skill",
-    "skill-setup": "skill",
-}
-
-
-def _patch_dspy_parallelizer():
-    """Monkey-patch DSPy's ParallelExecutor to handle exceptions properly.
-    
-    DSPy bug: parallelizer returns exception objects, but Evaluate only
-    handles None, causing 'too many values to unpack' errors.
-    """
-    try:
-        from dspy.utils import parallelizer
-
-        original_wrap = parallelizer.ParallelExecutor._wrap_function
-
-        def patched_wrap(self, user_function):
-            original_safe_func = original_wrap(self, user_function)
-            def safer_func(item):
-                result = original_safe_func(item)
-                if isinstance(result, Exception):
-                    return None
-                return result
-            return safer_func
-
-        parallelizer.ParallelExecutor._wrap_function = patched_wrap
-        logger.info("Applied DSPy parallelizer exception patch")
-    except Exception as e:
-        logger.warning("Failed to patch parallelizer: %s", e)
-
-
-def run_gepa(
-    target_name: str,
-    instruction: str,
-    target_type: str,
-    wiki_dir: str,
-    model_spec: str,
-    reflection_model_spec: str | None = None,
-    judge_model_spec: str | None = None,
-    auto: str = "light",
-    max_metric_calls: int | None = None,
-    num_threads: int = 2,
-    seed: int = 42,
-) -> dict:
-    """Run GEPA optimization on a single target.
-
-    Args:
-        target_name: Target identifier (e.g. 'capture-system-prompt')
-        instruction: Current instruction/prompt text
-        target_type: Type for dataset/metric (capture, ingest, query, etc.)
-        wiki_dir: Path to wiki directory
-        model_spec: Model for the proxy (e.g. 'anthropic/claude-sonnet-4-20250514')
-        reflection_model_spec: Model for GEPA reflection (defaults to model_spec)
-        auto: GEPA budget preset ('light', 'medium', 'heavy')
-        max_metric_calls: Override auto with explicit budget
-        num_threads: Parallel threads for evaluation
-        seed: Random seed
-
-    Returns:
-        Result dict with optimized instruction, scores, etc.
-    """
-    logger.info("Optimizing target: %s (type: %s)", target_name, target_type)
-
-    # 1. Create LM instances (teacher/student/judge pattern)
-    # Student LM: runs the proxy (generates wiki output from instruction)
-    student_lm = create_lm(model_spec, temperature=0.0)
-
-    # Teacher/Reflection LM: proposes mutations (needs creativity → temperature=1.0)
-    teacher_spec = reflection_model_spec or model_spec
-    teacher_lm = create_lm(teacher_spec, temperature=1.0, max_tokens=8000)
-
-    # Judge LM: scores candidates via metric (fast + cheap)
-    judge_spec = judge_model_spec or model_spec
-    judge_lm = create_lm(judge_spec, temperature=0.0)
-    set_judge_lm(judge_lm)
-
-    logger.info("Student: %s | Teacher: %s | Judge: %s", model_spec, teacher_spec, judge_spec)
-
-    # Configure DSPy with task LM
-    dspy.configure(lm=student_lm)
-
-    # 2. Build dataset
-    logger.info("Building dataset from wiki pages...")
-    try:
-        trainset, valset = build_dataset(
-            wiki_dir=wiki_dir,
-            target_type=target_type,
-            max_examples=50,
-            val_ratio=0.4,
-            seed=seed,
-        )
-    except ValueError as e:
-        logger.error("Dataset build failed: %s", e)
-        return {
-            "target": target_name,
-            "status": "error",
-            "error": str(e),
-        }
-
-    logger.info("Dataset: %d train, %d val examples", len(trainset), len(valset))
-
-    # 3. Create proxy from instruction
-    proxy = PromptProxy.from_instruction(instruction)
-
-    # 4. Compute baseline score
-    logger.info("Computing baseline...")
-    baseline_scores = []
-    for ex in trainset[:min(5, len(trainset))]:
-        try:
-            pred = proxy(task_context=ex.task_context)
-            result = wiki_quality_metric(ex, pred)
-            baseline_scores.append(float(result.score))
-        except Exception as e:
-            logger.debug("Baseline eval failed: %s", e)
-            baseline_scores.append(0.0)
-
-    baseline_score = sum(baseline_scores) / len(baseline_scores) if baseline_scores else 0.0
-    logger.info("Baseline score: %.3f", baseline_score)
-
-    # 5. Patch DSPy parallelizer
-    _patch_dspy_parallelizer()
-
-    # 6. Run GEPA
-    logger.info("Starting GEPA optimization (auto=%s)...", auto)
-
-    gepa_kwargs = {
-        "metric": wiki_quality_metric,
-        "reflection_lm": teacher_lm,
-        "reflection_minibatch_size": 3,
-        "candidate_selection_strategy": "pareto",
-        "skip_perfect_score": True,
-        "use_merge": True,
-        "num_threads": num_threads,
-        "failure_score": 0.0,
-        "perfect_score": 1.0,
-        "track_stats": True,
-        "seed": seed,
-    }
-
-    if max_metric_calls:
-        gepa_kwargs["max_metric_calls"] = max_metric_calls
-    else:
-        gepa_kwargs["auto"] = auto
-
-    try:
-        optimizer = dspy.GEPA(**gepa_kwargs)
-        optimized_proxy = optimizer.compile(
-            student=proxy,
-            trainset=trainset,
-            valset=valset,
-        )
-    except Exception as e:
-        logger.error("GEPA optimization failed: %s", e)
-        return {
-            "target": target_name,
-            "status": "error",
-            "error": str(e),
-            "baseline_score": baseline_score,
-        }
-
-    # 7. Extract results
-    evolved_instruction = extract_evolved_instruction(optimized_proxy)
-
-    # Get optimized score
-    optimized_score = baseline_score
-    if hasattr(optimized_proxy, "detailed_results") and optimized_proxy.detailed_results:
-        dr = optimized_proxy.detailed_results
-        if hasattr(dr, "val_aggregate_scores") and dr.val_aggregate_scores:
-            optimized_score = max(dr.val_aggregate_scores)
-    else:
-        # Fallback: manually evaluate
-        opt_scores = []
-        for ex in trainset[:min(5, len(trainset))]:
-            try:
-                pred = optimized_proxy(task_context=ex.task_context)
-                result = wiki_quality_metric(ex, pred)
-                opt_scores.append(float(result.score))
-            except Exception:
-                opt_scores.append(0.0)
-        if opt_scores:
-            optimized_score = sum(opt_scores) / len(opt_scores)
-
-    improvement = (
-        ((optimized_score - baseline_score) / baseline_score * 100)
-        if baseline_score > 0 else 0.0
-    )
-
-    logger.info(
-        "Optimization complete: %.3f → %.3f (%.1f%% improvement)",
-        baseline_score, optimized_score, improvement,
-    )
-
-    return {
-        "target": target_name,
-        "status": "success",
-        "baseline_score": round(baseline_score, 4),
-        "optimized_score": round(optimized_score, 4),
-        "improvement_pct": round(improvement, 1),
-        "optimized_instruction": evolved_instruction or instruction,
-        "original_instruction": instruction[:200] + "..." if len(instruction) > 200 else instruction,
-        "model": model_spec,
-        "auto": auto,
-        "train_size": len(trainset),
-        "val_size": len(valset),
-        "timestamp": datetime.now(timezone.utc).isoformat(),
-    }
-
-
-# -- CLI entry point -----------------------------------------------------------
-
-def main():
-    parser = argparse.ArgumentParser(description="GEPA optimizer for pi-para")
-    parser.add_argument("--targets-file", required=True, help="JSON file with targets to optimize")
-    parser.add_argument("--wiki-dir", default=os.path.expanduser("~/.pi/wiki"), help="Wiki directory")
-    parser.add_argument("--output", required=True, help="Output JSON file for results")
-    parser.add_argument("--model", default="anthropic/claude-sonnet-4-20250514", help="Student model (runs proxy)")
-    parser.add_argument("--reflection-model", default="anthropic/claude-opus-4-6", help="Teacher/reflection model (proposes mutations)")
-    parser.add_argument("--judge-model", default=None, help="Judge model (scores output, defaults to --model)")
-    parser.add_argument("--auto", default="light", choices=["light", "medium", "heavy"], help="GEPA budget")
-    parser.add_argument("--max-metric-calls", type=int, default=None, help="Override auto with explicit budget")
-    parser.add_argument("--threads", type=int, default=2, help="Parallel eval threads")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed")
-    parser.add_argument("--target", default=None, help="Optimize only this target (by name)")
-
-    args = parser.parse_args()
-
-    # Load targets
-    with open(args.targets_file, "r") as f:
-        targets_data = json.load(f)
-
-    targets = targets_data.get("targets", [])
-    if not targets:
-        logger.error("No targets in %s", args.targets_file)
-        sys.exit(1)
-
-    # Filter to specific target if requested
-    if args.target:
-        targets = [t for t in targets if t["name"] == args.target]
-        if not targets:
-            logger.error("Target '%s' not found", args.target)
-            sys.exit(1)
-
-    logger.info("Optimizing %d target(s) with model %s, auto=%s", len(targets), args.model, args.auto)
-
-    # Run GEPA for each target
-    results = []
-    for target in targets:
-        name = target["name"]
-        instruction = target["content"]
-        target_type = TARGET_TYPE_MAP.get(name, "capture")
-
-        result = run_gepa(
-            target_name=name,
-            instruction=instruction,
-            target_type=target_type,
-            wiki_dir=args.wiki_dir,
-            model_spec=args.model,
-            reflection_model_spec=args.reflection_model,
-            judge_model_spec=args.judge_model,
-            auto=args.auto,
-            max_metric_calls=args.max_metric_calls,
-            num_threads=args.threads,
-            seed=args.seed,
-        )
-        results.append(result)
-
-    # Write output
-    output_dir = os.path.dirname(args.output)
-    if output_dir:
-        os.makedirs(output_dir, exist_ok=True)
-
-    output_data = {
-        "results": results,
-        "model": args.model,
-        "auto": args.auto,
-        "wiki_dir": args.wiki_dir,
-        "timestamp": datetime.now(timezone.utc).isoformat(),
-    }
-
-    with open(args.output, "w") as f:
-        json.dump(output_data, f, indent=2)
-
-    logger.info("Results written to %s", args.output)
-
-    # Print summary
-    print("\n" + "=" * 60)
-    print("GEPA Optimization Summary")
-    print("=" * 60)
-    for r in results:
-        status = r.get("status", "unknown")
-        if status == "success":
-            print(
-                f"  {r['target']}: {r['baseline_score']:.3f} → {r['optimized_score']:.3f} "
-                f"({r['improvement_pct']:+.1f}%)"
-            )
-        else:
-            print(f"  {r['target']}: ERROR — {r.get('error', 'unknown')}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()