devsper 2.7.0__tar.gz → 2.7.2__tar.gz

{devsper-2.7.0 → devsper-2.7.2}/CHANGELOG.md

@@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.7.2] — 2026-04-07
+
+### Changed
+
+- **Prompt optimizer default** — Auto-detect replaces hard-coded `noop` default. Factory resolves to `dspy` if `dspy-ai` is importable, else `gepa` (built-in evolutionary loop, no extra install required). Set `provider = "noop"` explicitly to disable.
+- **`PromptOptimizerConfig.provider`** — Default changed from `"noop"` to `""` (auto-detect).
+
+## [2.7.1] — 2026-04-07
+
+### Added
+
+- **Eval harness** — New `devsper/evals/` module: `EvalCase`, `EvalDataset` (JSONL-backed), `EvalRunner` (async, bounded concurrency), `EvalSummary`, and `MetricFn` protocol. Built-in metrics: `exact_match`, `contains`, `regex_match`, `word_overlap`, `non_empty`, `llm_judge(model=)`.
+- **Prompt optimizer abstraction** — `PromptOptimizerBackend` ABC mirroring the `MemoryBackend` / `LLMBackend` hot-swap pattern. Active backend resolved from `DEVSPER_PROMPT_OPTIMIZER` env var → `[prompt_optimizer] provider` config → default `"noop"`.
+- **DSPy backend** (`devsper[dspy]`) — `DSPyBackend` compiles few-shot examples into an optimized system prompt using `BootstrapFewShot`, `MIPROv2`, or `BootstrapFewShotWithRandomSearch`. Configured via `[prompt_optimizer] dspy_optimizer`, `max_demos`, `num_candidates`.
+- **GEPA backend** (`devsper[gepa]`) — `GEPABackend` runs an evolutionary prompt optimization loop. Uses the `gepa` library when installed; falls back to a built-in LLM-driven mutation loop so it works without any extra dependencies.
+- **OpenEvals metric adapter** (`devsper[openevals]`) — `openevals_metric(name)` wraps any OpenEvals LLM-as-judge evaluator as a `MetricFn`. Prebuilt names: `correctness`, `conciseness`, `groundedness`, `relevance`. Accessible via `get_metric("openevals:correctness")`. Falls back to built-in `llm_judge` when the package is absent.
+- **`devsper eval` CLI** — Three subcommands: `eval stub` (generate JSONL stub datasets per role), `eval run` (score a dataset, optional `--optimize` flag, `--optimizer dspy|gepa`), `eval results` (list persisted result files).
+- **Config sections** — `[prompt_optimizer]` and `[evals]` added to `devsperConfigModel`.
+- **Optimized prompt persistence** — `EvalRunner` saves results to `.devsper/optimized_prompts/{role}.json`. `get_role_config()` auto-loads these on the next run.
+- **`devsper[evals]`, `devsper[dspy]`, `devsper[gepa]`, `devsper[openevals]` extras.**
+- **pytest `live` marker** — Gates tests that require real API keys.
+
+### Changed
+
+- **`get_metric(name)`** extended: now accepts `"openevals:<evaluator>"` in addition to built-in names.
+- **`get_role_config()`** — Checks `.devsper/optimized_prompts/{role}.json` and uses the optimized prefix if present.
+
 ## [2.7.0] — 2026-04-07
 
 ### Added

{devsper-2.7.0 → devsper-2.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devsper
-Version: 2.7.0
+Version: 2.7.2
 Summary: Orchestrate distributed swarms of AI agents that collaboratively solve complex tasks.
 Project-URL: Homepage, https://github.com/devsper-com/runtime
 Project-URL: Documentation, https://docs.devsper.com
@@ -57,9 +57,14 @@ Requires-Dist: mkdocs<2,>=1.6.0; extra == 'docs'
 Provides-Extra: document
 Requires-Dist: docproc>=2.0.0; extra == 'document'
 Requires-Dist: python-docx>=1.0.0; extra == 'document'
+Provides-Extra: dspy
+Requires-Dist: dspy-ai>=2.4; extra == 'dspy'
 Provides-Extra: embeddings
 Requires-Dist: sentence-transformers>=3.0.0; extra == 'embeddings'
+Provides-Extra: evals
 Provides-Extra: explainability
+Provides-Extra: gepa
+Requires-Dist: gepa>=0.1; extra == 'gepa'
 Provides-Extra: hitl
 Requires-Dist: httpx>=0.27; extra == 'hitl'
 Provides-Extra: langgraph
@@ -68,6 +73,8 @@ Provides-Extra: mcp
 Requires-Dist: anyio>=4.0; extra == 'mcp'
 Requires-Dist: httpx>=0.27; extra == 'mcp'
 Provides-Extra: ollama
+Provides-Extra: openevals
+Requires-Dist: openevals>=0.0.6; extra == 'openevals'
 Provides-Extra: redis-memory
 Requires-Dist: hiredis>=2.3.0; extra == 'redis-memory'
 Requires-Dist: redis>=5.0.0; extra == 'redis-memory'

{devsper-2.7.0 → devsper-2.7.2}/devsper/agents/roles.py

@@ -105,10 +105,34 @@ ROLE_CONFIGS: dict[str, RoleConfig] = {
 
 
 def get_role_config(role: str | None) -> RoleConfig:
-    """Return config for the given role, or default if unknown/None."""
-    if role and role in ROLE_CONFIGS:
-        return ROLE_CONFIGS[role]
-    return ROLE_CONFIGS[DEFAULT_ROLE]
+    """Return config for the given role, or default if unknown/None.
+
+    If an optimized prompt exists at .devsper/optimized_prompts/{role}.json
+    (written by EvalRunner after prompt optimization), it is used in place of
+    the hardcoded prompt_prefix.
+    """
+    base = ROLE_CONFIGS.get(role or DEFAULT_ROLE, ROLE_CONFIGS[DEFAULT_ROLE]) if role else ROLE_CONFIGS[DEFAULT_ROLE]
+    optimized = _load_optimized_prompt(base.name)
+    if optimized:
+        from dataclasses import replace
+        return replace(base, prompt_prefix=optimized)
+    return base
+
+
+def _load_optimized_prompt(role: str) -> str | None:
+    """Load persisted optimized prompt for a role, if it exists."""
+    import json
+    from pathlib import Path
+
+    p = Path(".devsper/optimized_prompts") / f"{role}.json"
+    if not p.exists():
+        return None
+    try:
+        data = json.loads(p.read_text())
+        prompt = data.get("prompt_prefix", "")
+        return prompt if prompt else None
+    except Exception:
+        return None
 
 
 # Keywords to infer role from task description

{devsper-2.7.0 → devsper-2.7.2}/devsper/cli/main.py

@@ -145,7 +145,7 @@ def _run_platform_once(task: str, args: object) -> tuple[int, dict, float]:
     api = _platform_api_builder()
     if not api.enabled():
         print(
-            "Platform routing is not configured. Run `devsper platform connect` first.",
+            "Cloud routing is not configured. Run `devsper platform connect` first.",
             file=sys.stderr,
         )
         return 2, {}, 0.0
@@ -2721,6 +2721,164 @@ def _run_version(args: object) -> int:
     return 0
 
 
+def _run_eval(args: object) -> int:
+    """Eval harness: run dataset, score results, optionally optimize prompts."""
+    import asyncio
+    import json
+    from pathlib import Path
+
+    eval_cmd = getattr(args, "eval_cmd", None)
+
+    if eval_cmd == "stub" or eval_cmd is None and not hasattr(args, "dataset"):
+        # Generate stub dataset
+        from devsper.evals.dataset import EvalDataset
+
+        role = getattr(args, "role", "general")
+        n = getattr(args, "n", 5)
+        out = getattr(args, "out", None)
+        dataset = EvalDataset.stub(role=role, n=n)
+        if out:
+            dataset.save(out)
+            print(f"Stub dataset ({len(dataset)} cases) written to {out}")
+        else:
+            for case in dataset:
+                print(json.dumps(case.to_dict()))
+        return 0
+
+    if eval_cmd == "results":
+        from devsper.config import get_config
+
+        try:
+            results_dir = Path(getattr(args, "dir", None) or get_config().evals.results_dir)
+        except Exception:
+            results_dir = Path(".devsper/eval_results")
+        if not results_dir.exists():
+            print(f"No results found in {results_dir}")
+            return 0
+        files = sorted(results_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+        if not files:
+            print("No eval result files found.")
+            return 0
+        for f in files[:20]:
+            try:
+                data = json.loads(f.read_text())
+                print(
+                    f"  {f.name}  role={data.get('role', '?')}  "
+                    f"pass_rate={data.get('pass_rate', '?')}  "
+                    f"mean_score={data.get('mean_score', '?')}"
+                )
+            except Exception:
+                print(f"  {f.name}")
+        return 0
+
+    # eval_cmd == "run"
+    from devsper.evals.dataset import EvalDataset
+    from devsper.evals.metrics import get_metric
+    from devsper.evals.runner import EvalRunner
+    from devsper.config import get_config
+
+    try:
+        cfg = get_config()
+    except Exception:
+        from devsper.config.schema import devsperConfigModel
+        cfg = devsperConfigModel()
+
+    dataset_path = getattr(args, "dataset", None)
+    if not dataset_path:
+        print("Error: --dataset is required for 'eval run'")
+        return 1
+
+    dataset = EvalDataset.load(dataset_path)
+    role = getattr(args, "role", None)
+    metric_name = getattr(args, "metric", None) or cfg.evals.default_metric
+    threshold = getattr(args, "threshold", None) or cfg.evals.pass_threshold
+    concurrency = getattr(args, "concurrency", None) or cfg.evals.concurrency
+    do_optimize = getattr(args, "optimize", False)
+    optimizer_override = getattr(args, "optimizer", None)
+    out_path = getattr(args, "out", None)
+
+    metric = get_metric(metric_name)
+
+    # Build optimizer if requested
+    optimizer = None
+    if do_optimize:
+        from devsper.prompt_optimizer.factory import get_prompt_optimizer, reset_prompt_optimizer
+
+        if optimizer_override:
+            import os
+            os.environ["DEVSPER_PROMPT_OPTIMIZER"] = optimizer_override
+            reset_prompt_optimizer()
+        optimizer = get_prompt_optimizer(cfg)
+
+    # Build a minimal agent for evaluation
+    from devsper.agents.agent import Agent
+
+    agent = Agent(model_name=cfg.models.worker, use_tools=False)
+
+    runner = EvalRunner(
+        agent=agent,
+        metric=metric,
+        pass_threshold=threshold,
+        concurrency=concurrency,
+        optimize_after=do_optimize,
+        optimizer=optimizer,
+    )
+
+    try:
+        summary = asyncio.run(runner.run_async(dataset, role=role))
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        summary = loop.run_until_complete(runner.run_async(dataset, role=role))
+
+    # Print summary
+    try:
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+        console.print(
+            f"\n[bold]Eval Results[/bold]  role=[cyan]{summary.role}[/cyan]  "
+            f"metric=[cyan]{summary.metric_name}[/cyan]  "
+            f"optimizer=[cyan]{summary.optimizer_backend}[/cyan]"
+        )
+        console.print(
+            f"  Passed: [green]{summary.passed}[/green]/{summary.total}  "
+            f"Pass rate: [bold]{summary.pass_rate:.1%}[/bold]  "
+            f"Mean score: [bold]{summary.mean_score:.3f}[/bold]\n"
+        )
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("ID", style="dim")
+        table.add_column("Task", max_width=40)
+        table.add_column("Score")
+        table.add_column("Pass")
+        for r in summary.results:
+            color = "green" if r.passed else "red"
+            table.add_row(
+                r.case.id,
+                r.case.task[:40],
+                f"{r.score:.2f}",
+                f"[{color}]{'✓' if r.passed else '✗'}[/{color}]",
+            )
+        console.print(table)
+    except ImportError:
+        print(f"\nEval: role={summary.role} metric={summary.metric_name}")
+        print(f"  {summary.passed}/{summary.total} passed ({summary.pass_rate:.1%})")
+        print(f"  Mean score: {summary.mean_score:.3f}")
+
+    # Persist results
+    results_dir = Path(cfg.evals.results_dir)
+    results_dir.mkdir(parents=True, exist_ok=True)
+    ts = __import__("datetime").datetime.now().strftime("%Y%m%d_%H%M%S")
+    result_file = results_dir / f"eval_{summary.role}_{ts}.json"
+    result_file.write_text(summary.to_json())
+    print(f"\nResults saved to {result_file}")
+
+    if out_path:
+        Path(out_path).write_text(summary.to_json())
+
+    return 0 if summary.pass_rate >= threshold else 1
+
+
 def _run_health(args: object) -> int:
     """Run health checks. Exit 0 if healthy, 1 otherwise. Print ✓/✗ per check."""
     import asyncio
@@ -2811,7 +2969,7 @@ def _run_upgrade(args: object) -> int:
 
 
 def _run_cloud_dispatch(args: object) -> int:
-    """Devsper Platform (cloud): login, run, status, logs."""
+    """Devsper Cloud: login, run, status, logs."""
     cmd = getattr(args, "cloud_cmd", None)
     if not cmd:
         return 0
@@ -4082,6 +4240,64 @@ Examples:
     )
     observe_parser.set_defaults(func=lambda a: _run_observe(a.port, a.db))
 
+    eval_parser = subparsers.add_parser(
+        "eval",
+        help="Eval harness and prompt optimization",
+        description="Run evals against a JSONL dataset and optionally optimize prompts.",
+        epilog="""
+Examples:
+  devsper eval run --dataset evals.jsonl --metric contains
+  devsper eval run --dataset evals.jsonl --role research --optimize --optimizer dspy
+  devsper eval stub --role research --out evals.jsonl
+  devsper eval results
+""",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    eval_sub = eval_parser.add_subparsers(dest="eval_cmd", help="Subcommand")
+
+    eval_run_p = eval_sub.add_parser("run", help="Run eval dataset")
+    eval_run_p.add_argument("--dataset", required=True, help="Path to JSONL dataset")
+    eval_run_p.add_argument("--role", default=None, help="Filter to this agent role")
+    eval_run_p.add_argument(
+        "--metric",
+        default=None,
+        help="Metric name: exact_match | contains | regex_match | word_overlap | llm_judge (default: config)",
+    )
+    eval_run_p.add_argument(
+        "--threshold", type=float, default=None, help="Pass threshold (default: config)"
+    )
+    eval_run_p.add_argument(
+        "--optimize",
+        action="store_true",
+        help="Run prompt optimization after eval using the configured optimizer",
+    )
+    eval_run_p.add_argument(
+        "--optimizer",
+        default=None,
+        help="Override optimizer backend: noop | dspy | gepa",
+    )
+    eval_run_p.add_argument(
+        "--concurrency", type=int, default=None, help="Parallel eval cases"
+    )
+    eval_run_p.add_argument("--out", default=None, help="Save JSON results to this path")
+    eval_run_p.set_defaults(eval_cmd="run")
+
+    eval_stub_p = eval_sub.add_parser("stub", help="Generate a stub dataset")
+    eval_stub_p.add_argument(
+        "--role", default="general", help="Agent role (research/code/analysis/general)"
+    )
+    eval_stub_p.add_argument("-n", type=int, default=5, help="Number of examples")
+    eval_stub_p.add_argument(
+        "--out", default=None, help="Output JSONL path (default: prints to stdout)"
+    )
+    eval_stub_p.set_defaults(eval_cmd="stub")
+
+    eval_results_p = eval_sub.add_parser("results", help="List recent eval result files")
+    eval_results_p.add_argument("--dir", default=None, help="Results directory")
+    eval_results_p.set_defaults(eval_cmd="results")
+
+    eval_parser.set_defaults(func=_run_eval)
+
     health_parser = subparsers.add_parser(
         "health",
         help="Health and readiness check",

{devsper-2.7.0 → devsper-2.7.2}/devsper/config/schema.py

@@ -309,6 +309,32 @@ class HitlConfig(BaseModel):
     policies: list[HitlPolicyConfig] = Field(default_factory=list)
 
 
+class PromptOptimizerConfig(BaseModel):
+    """v2.7: Prompt optimization backend config.
+
+    provider: "noop" | "dspy" | "gepa"  (override with DEVSPER_PROMPT_OPTIMIZER env var)
+    """
+
+    provider: str = ""   # "" = auto (dspy if installed, else gepa). "noop" disables.
+    # DSPy-specific
+    dspy_optimizer: str = "bootstrap"  # "bootstrap" | "mipro" | "bootstrap_random"
+    max_demos: int = 4
+    num_candidates: int = 10           # MIPROv2 only
+    # GEPA-specific
+    population_size: int = 5
+    n_iterations: int = 10
+
+
+class EvalConfig(BaseModel):
+    """v2.7: Eval harness config."""
+
+    dataset_dir: str = ".devsper/evals"   # default location for JSONL datasets
+    results_dir: str = ".devsper/eval_results"
+    pass_threshold: float = 0.5
+    concurrency: int = 4
+    default_metric: str = "contains"      # metric name from devsper.evals.metrics
+
+
 class devsperConfigModel(BaseModel):
     """Full resolved configuration with Pydantic validation."""
 
@@ -333,6 +359,8 @@ class devsperConfigModel(BaseModel):
     sandbox: SandboxConfig = Field(default_factory=SandboxConfig)
     compliance: ComplianceConfig = Field(default_factory=ComplianceConfig)
     hitl: HitlConfig = Field(default_factory=HitlConfig)
+    prompt_optimizer: PromptOptimizerConfig = Field(default_factory=PromptOptimizerConfig)
+    evals: EvalConfig = Field(default_factory=EvalConfig)
 
     # Backward-compat aliases (property-style access from old devsperConfig)
     @property

devsper-2.7.2/devsper/evals/__init__.py

@@ -0,0 +1,19 @@
+"""
+devsper evals — eval harness and prompt optimization integration.
+"""
+
+from devsper.evals.types import EvalCase, EvalResult, EvalSummary, MetricFn
+from devsper.evals.dataset import EvalDataset
+from devsper.evals.metrics import get_metric, BUILTIN_METRICS
+from devsper.evals.runner import EvalRunner
+
+__all__ = [
+    "EvalCase",
+    "EvalResult",
+    "EvalSummary",
+    "EvalDataset",
+    "EvalRunner",
+    "MetricFn",
+    "get_metric",
+    "BUILTIN_METRICS",
+]

devsper-2.7.2/devsper/evals/dataset.py

@@ -0,0 +1,107 @@
+"""
+JSONL-backed eval dataset: load, save, and generate stub datasets.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from pathlib import Path
+
+from devsper.evals.types import EvalCase
+
+
+class EvalDataset:
+    """Ordered collection of EvalCases backed by a JSONL file."""
+
+    def __init__(self, cases: list[EvalCase] | None = None, name: str = "dataset"):
+        self.cases: list[EvalCase] = cases or []
+        self.name = name
+
+    def __len__(self) -> int:
+        return len(self.cases)
+
+    def __iter__(self):
+        return iter(self.cases)
+
+    def add(self, case: EvalCase) -> None:
+        self.cases.append(case)
+
+    def filter_by_role(self, role: str) -> "EvalDataset":
+        return EvalDataset(
+            [c for c in self.cases if c.role == role],
+            name=f"{self.name}[{role}]",
+        )
+
+    @classmethod
+    def load(cls, path: str | Path) -> "EvalDataset":
+        """Load from a JSONL file. Each line is a JSON object (EvalCase dict)."""
+        p = Path(path)
+        if not p.exists():
+            raise FileNotFoundError(f"Dataset not found: {path}")
+        cases = []
+        with p.open() as f:
+            for i, line in enumerate(f):
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                try:
+                    d = json.loads(line)
+                    if "id" not in d:
+                        d["id"] = str(uuid.uuid4())[:8]
+                    cases.append(EvalCase.from_dict(d))
+                except json.JSONDecodeError as e:
+                    raise ValueError(f"Bad JSON on line {i + 1} of {path}: {e}") from e
+        return cls(cases, name=p.stem)
+
+    def save(self, path: str | Path) -> None:
+        """Write to a JSONL file (overwrites)."""
+        p = Path(path)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        with p.open("w") as f:
+            for case in self.cases:
+                f.write(json.dumps(case.to_dict()) + "\n")
+
+    @classmethod
+    def from_dicts(cls, records: list[dict], name: str = "dataset") -> "EvalDataset":
+        cases = []
+        for i, d in enumerate(records):
+            if "id" not in d:
+                d["id"] = str(i)
+            cases.append(EvalCase.from_dict(d))
+        return cls(cases, name=name)
+
+    @classmethod
+    def stub(cls, role: str = "general", n: int = 5) -> "EvalDataset":
+        """Generate a minimal stub dataset for smoke-testing."""
+        stubs = {
+            "research": [
+                ("What is transformer architecture?", "attention mechanism"),
+                ("Summarize BERT vs GPT", "bidirectional"),
+                ("What year was AlexNet published?", "2012"),
+                ("Who invented backpropagation?", "Rumelhart"),
+                ("What is RLHF?", "reinforcement learning from human feedback"),
+            ],
+            "code": [
+                ("Write a Python function to reverse a string", "def"),
+                ("What is a decorator in Python?", "wraps"),
+                ("Implement binary search", "mid"),
+                ("What does __init__ do?", "constructor"),
+                ("How do you open a file in Python?", "open("),
+            ],
+            "analysis": [
+                ("What is mean absolute error?", "absolute"),
+                ("Explain p-value", "null hypothesis"),
+                ("What is overfitting?", "training data"),
+                ("Define precision and recall", "true positive"),
+                ("What is cross-validation?", "fold"),
+            ],
+        }
+        pairs = stubs.get(role, [
+            (f"Task {i}", f"expected {i}") for i in range(n)
+        ])[:n]
+        cases = [
+            EvalCase(id=str(i), task=t, expected=e, role=role)
+            for i, (t, e) in enumerate(pairs)
+        ]
+        return cls(cases, name=f"stub_{role}")