agentforge-py 0.2.1__py3-none-any.whl

agentforge/cli/upgrade_cmd.py

@@ -0,0 +1,230 @@
+"""`agentforge upgrade/fork/unfork/status` commands (feat-011 chunks 4-5).
+
+- **upgrade**: wraps Copier's `copier update` for the linked
+  template; refreshes the managed-files lock.
+- **fork**: strip the framework marker from a file + flag it as
+  forked in the lock. Future upgrades skip it.
+- **unfork**: restore from the template (lossy — overwrites local
+  edits). Re-runs the per-file render and updates the lock.
+- **status**: walks the lock and prints managed / forked / drifted
+  / missing per file.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import yaml
+from agentforge_core.production.exceptions import ModuleError
+
+from agentforge.cli._scaffold_state import (
+    answers_path,
+    file_status,
+    hash_content,
+    marker_for,
+    read_lock,
+    strip_marker,
+    write_lock,
+)
+
+
+def register_upgrade_cmds(sub: argparse._SubParsersAction) -> None:  # type: ignore[type-arg]
+    """Attach upgrade / fork / unfork / status to the parent
+    subparser action."""
+    upgrade = sub.add_parser(
+        "upgrade",
+        help="Pull framework updates into this agent (three-way merge).",
+    )
+    upgrade.add_argument("--to", default=None, help="Target version (default: latest).")
+    upgrade.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would change without writing.",
+    )
+    upgrade.set_defaults(_handler=_run_upgrade)
+
+    fork = sub.add_parser("fork", help="Claim a managed file — future upgrades skip it.")
+    fork.add_argument("path", help="Path to the file to fork (relative to cwd).")
+    fork.set_defaults(_handler=_run_fork)
+
+    unfork = sub.add_parser("unfork", help="Restore a forked file to the template version.")
+    unfork.add_argument("path", help="Path to the file to unfork.")
+    unfork.set_defaults(_handler=_run_unfork)
+
+    status = sub.add_parser(
+        "status",
+        help="Show managed / forked / drifted files in this agent.",
+    )
+    status.set_defaults(_handler=_run_status)
+
+
+# ----------------------------------------------------------------------
+# upgrade
+# ----------------------------------------------------------------------
+
+
+def _run_upgrade(
+    args: argparse.Namespace,
+    *,
+    cwd: Path | None = None,
+) -> int:
+    """Run `copier update` against the linked template + refresh lock.
+
+    Copier handles the three-way merge against the answer file's
+    recorded template-version. We only handle the lock refresh
+    afterwards.
+    """
+    work_dir = cwd if cwd is not None else Path.cwd()
+    if not answers_path(work_dir).exists():
+        sys.stderr.write(
+            "No .agentforge-state/answers.yml; this directory wasn't scaffolded by "
+            "`agentforge new`. Nothing to upgrade.\n"
+        )
+        return 1
+
+    if args.dry_run:
+        sys.stdout.write("  → dry-run: not actually running copier update\n")
+        return 0
+
+    try:
+        _run_copier_update(work_dir, to=args.to)
+    except ModuleError as exc:
+        sys.stderr.write(f"upgrade failed: {exc}\n")
+        return 1
+
+    # Refresh the lock: re-hash every still-managed file against its
+    # new content. Forked entries stay flagged.
+    lock = read_lock(work_dir)
+    new_lock: dict[str, dict[str, object]] = {}
+    for rel, entry in lock.items():
+        path = work_dir / rel
+        if not path.exists():
+            continue
+        if entry.get("forked"):
+            new_lock[rel] = entry
+            continue
+        try:
+            content = path.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, OSError):
+            new_lock[rel] = entry
+            continue
+        # Strip marker before hashing.
+        from agentforge.cli._scaffold_state import _strip_marker_for_hash  # noqa: PLC0415
+
+        body = _strip_marker_for_hash(content)
+        new_lock[rel] = {**entry, "hash": hash_content(body)}
+    write_lock(work_dir, new_lock)
+    sys.stdout.write("  → upgrade complete; lock refreshed.\n")
+    return 0
+
+
+def _run_copier_update(cwd: Path, *, to: str | None) -> None:
+    from copier import run_update  # noqa: PLC0415
+
+    try:
+        run_update(
+            dst_path=str(cwd),
+            vcs_ref=to or "HEAD",
+            defaults=True,
+            overwrite=True,
+            quiet=False,
+        )
+    except Exception as exc:
+        raise ModuleError(f"copier update failed: {exc}") from exc
+
+
+# ----------------------------------------------------------------------
+# fork / unfork
+# ----------------------------------------------------------------------
+
+
+def _run_fork(args: argparse.Namespace, *, cwd: Path | None = None) -> int:
+    work_dir = cwd if cwd is not None else Path.cwd()
+    rel = args.path
+    lock = read_lock(work_dir)
+    if rel not in lock:
+        sys.stderr.write(f"{rel} is not in the managed-files lock; nothing to fork.\n")
+        return 1
+    target = work_dir / rel
+    strip_marker(target)
+    lock[rel] = {**lock[rel], "forked": True}
+    write_lock(work_dir, lock)
+    sys.stdout.write(f"  → forked {rel}. Future upgrades will skip it.\n")
+    return 0
+
+
+def _run_unfork(args: argparse.Namespace, *, cwd: Path | None = None) -> int:
+    work_dir = cwd if cwd is not None else Path.cwd()
+    rel = args.path
+    lock = read_lock(work_dir)
+    if rel not in lock:
+        sys.stderr.write(f"{rel} is not in the managed-files lock.\n")
+        return 1
+    if not lock[rel].get("forked"):
+        sys.stderr.write(f"{rel} is not forked.\n")
+        return 1
+    # Flip the flag and re-prepend the marker. We can't restore the
+    # full template-version content without re-running Copier; for
+    # now, just clear the forked flag and recompute the hash. The
+    # next `agentforge upgrade` will re-render the file.
+    target = work_dir / rel
+    if target.exists():
+        try:
+            content = target.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, OSError):
+            content = ""
+        from agentforge.cli._scaffold_state import _strip_marker_for_hash  # noqa: PLC0415
+
+        body = _strip_marker_for_hash(content)
+        marker = marker_for(
+            target.suffix,
+            lock[rel].get("source_module", "template:unknown"),
+            lock[rel].get("source_version", "0"),
+            hash_content(body)[:12],
+        )
+        if marker:
+            target.write_text(marker + "\n" + body, encoding="utf-8")
+        lock[rel] = {**lock[rel], "forked": False, "hash": hash_content(body)}
+    else:
+        lock[rel] = {**lock[rel], "forked": False}
+    write_lock(work_dir, lock)
+    sys.stdout.write(f"  → unforked {rel}. Run `agentforge upgrade` to pull template content.\n")
+    return 0
+
+
+# ----------------------------------------------------------------------
+# status
+# ----------------------------------------------------------------------
+
+
+def _run_status(args: argparse.Namespace, *, cwd: Path | None = None) -> int:
+    del args
+    work_dir = cwd if cwd is not None else Path.cwd()
+    lock = read_lock(work_dir)
+    if not lock:
+        sys.stdout.write("No managed-files lock; this directory wasn't scaffolded.\n")
+        return 0
+
+    by_status: dict[str, list[str]] = {"managed": [], "forked": [], "drifted": [], "missing": []}
+    for rel, entry in sorted(lock.items()):
+        status = file_status(work_dir, rel, entry)
+        by_status[status].append(rel)
+
+    for label in ("managed", "forked", "drifted", "missing"):
+        files = by_status[label]
+        if not files:
+            continue
+        sys.stdout.write(f"\n{label.upper()} ({len(files)})\n")
+        for rel in files:
+            sys.stdout.write(f"  {rel}\n")
+    return 0
+
+
+__all__ = ["register_upgrade_cmds"]
+
+
+# Suppress unused-import warning in module-level imports the file
+# uses transitively.
+_ = yaml

agentforge/config/__init__.py

@@ -0,0 +1,45 @@
+"""Configuration loader for `agentforge.yaml` — re-export from core.
+
+feat-012 moved the canonical schema + loader to `agentforge-core`
+so the resolver can compose module-side Pydantic schemas without
+importing the runtime package. This module stays as a re-export
+for the historical `from agentforge.config import ...` path.
+"""
+
+from __future__ import annotations
+
+from agentforge_core.config import (
+    AgentConfig,
+    AgentForgeConfig,
+    BudgetConfig,
+    EvaluatorEntry,
+    GraphModuleConfig,
+    LoggingConfig,
+    MemoryModuleConfig,
+    ModuleEntry,
+    ModulesConfig,
+    ObservabilityEntry,
+    OutputConfig,
+    ProviderConfig,
+    RetrieverModuleConfig,
+    load_config,
+    parse_overrides,
+)
+
+__all__ = [
+    "AgentConfig",
+    "AgentForgeConfig",
+    "BudgetConfig",
+    "EvaluatorEntry",
+    "GraphModuleConfig",
+    "LoggingConfig",
+    "MemoryModuleConfig",
+    "ModuleEntry",
+    "ModulesConfig",
+    "ObservabilityEntry",
+    "OutputConfig",
+    "ProviderConfig",
+    "RetrieverModuleConfig",
+    "load_config",
+    "parse_overrides",
+]

agentforge/eval/__init__.py

@@ -0,0 +1,18 @@
+"""Deterministic evaluators shipped in `agentforge` (feat-006).
+
+Zero-cost graders that don't call an LLM — safe to run on every
+output. LLM-judge graders ship separately in `agentforge-eval-geval`.
+
+Each grader is constructible directly (`Coverage(reference={...})`),
+or addressable by name through the resolver (`"coverage"` etc.) when
+the runtime is asked to look up a grader by string.
+"""
+
+from __future__ import annotations
+
+from agentforge.eval.consistency import Consistency
+from agentforge.eval.coverage import Coverage
+from agentforge.eval.format_compliance import FormatCompliance
+from agentforge.eval.regression import RegressionVsBaseline
+
+__all__ = ["Consistency", "Coverage", "FormatCompliance", "RegressionVsBaseline"]

agentforge/eval/consistency.py

@@ -0,0 +1,107 @@
+"""`Consistency` — deterministic grader for "same input → same output".
+
+Re-runs the task N times via a caller-supplied async function and
+scores the agreement of the N outputs against the original output.
+The re-run function is the seam — for unit tests it can be a
+scripted-response function; in production it typically wraps another
+`Agent.run(task)` call.
+
+The grader declares `cost_estimate_usd = 0.0` against the evaluator
+budget gate (it doesn't bill itself), but the re-run function calls
+the LLM and bills against the run's `BudgetPolicy` like any other
+agent call. The caller is responsible for ensuring the runner
+respects the same budget if they want a unified cost cap.
+
+Score = fraction of re-runs whose output matches the original. The
+match function defaults to strict equality; pass a custom
+`matcher` for fuzzy comparison (cosine similarity, normalised
+string compare, etc.).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from typing import Any, ClassVar
+
+from agentforge_core.contracts.evaluator import EvalResult, Evaluator
+
+Runner = Callable[[str], Awaitable[Any]]
+"""Async function that re-executes the task and returns the new output."""
+
+Matcher = Callable[[Any, Any], bool]
+"""Equality check between the original output and one re-run output."""
+
+
+class Consistency(Evaluator):
+    """Score the fraction of N re-runs that match the original output."""
+
+    name: ClassVar[str] = "consistency"
+    # The grader itself does not call an LLM; re-runs bill against the
+    # outer run's BudgetPolicy via the caller-supplied runner. The
+    # evaluator gate treats this as $0 so it isn't skipped even on a
+    # tight budget; budget exhaustion will manifest as the runner
+    # itself raising BudgetExceeded.
+    cost_estimate_usd: ClassVar[float] = 0.0
+
+    def __init__(
+        self,
+        *,
+        runner: Runner,
+        n_samples: int = 3,
+        matcher: Matcher | None = None,
+    ) -> None:
+        if n_samples < 1:
+            raise ValueError(f"n_samples must be >= 1; got {n_samples}")
+        self._runner = runner
+        self._n = n_samples
+        self._matcher: Matcher = matcher if matcher is not None else _strict_eq
+
+    async def evaluate(self, finding: Any, context: dict[str, Any]) -> EvalResult:
+        task = context.get("task")
+        if not isinstance(task, str):
+            return EvalResult(
+                evaluator=self.name,
+                score=0.0,
+                label="fail",
+                reasoning="context['task'] missing or not a string",
+            )
+        original = finding.output if hasattr(finding, "output") else finding
+
+        agreements = 0
+        rerun_outputs: list[Any] = []
+        for i in range(self._n):
+            try:
+                replay = await self._runner(task)
+            except Exception as exc:
+                return EvalResult(
+                    evaluator=self.name,
+                    score=0.0,
+                    label="fail",
+                    reasoning=f"re-run {i + 1}/{self._n} raised {type(exc).__name__}: {exc}",
+                    raw={"rerun_outputs": rerun_outputs},
+                )
+            rerun_outputs.append(replay)
+            if self._matcher(original, replay):
+                agreements += 1
+
+        score = agreements / self._n
+        label = "pass" if agreements == self._n else "warn" if agreements > 0 else "fail"
+        return EvalResult(
+            evaluator=self.name,
+            score=score,
+            label=label,
+            reasoning=f"{agreements}/{self._n} re-runs matched the original",
+            raw={
+                "n_samples": self._n,
+                "agreements": agreements,
+                "original": original,
+                "rerun_outputs": rerun_outputs,
+            },
+        )
+
+
+def _strict_eq(a: Any, b: Any) -> bool:
+    return bool(a == b)
+
+
+__all__ = ["Consistency", "Matcher", "Runner"]

agentforge/eval/coverage.py

@@ -0,0 +1,100 @@
+"""`Coverage` — deterministic grader for "what fraction of expected items did the agent find?"
+
+The reference set is supplied at construction; the grader extracts the
+agent's items from `RunResult.output` (string match by default; callable
+override for structural extraction) and computes
+`score = |intersection| / |reference|` clamped to `[0, 1]`.
+
+Use for code-review agents (did the agent flag every known issue?),
+RAG agents (did the answer cite every required source?), and any
+task where ground truth is "should mention exactly these things".
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from typing import Any, ClassVar
+
+from agentforge_core.contracts.evaluator import EvalResult, Evaluator
+
+
+class Coverage(Evaluator):
+    """Fraction of expected items present in the agent's output.
+
+    Construction:
+
+        Coverage(reference={"sql injection", "xss", "csrf"})
+
+        # Or with a custom extractor (e.g. structured output):
+        Coverage(
+            reference={"item-1", "item-2"},
+            extractor=lambda out: set(out["found"]),
+        )
+
+    By default, items are matched against the output by case-
+    insensitive substring containment. Pass `extractor` for structural
+    matching (e.g. read a list from `output["findings"]`).
+    """
+
+    name: ClassVar[str] = "coverage"
+    cost_estimate_usd: ClassVar[float] = 0.0
+
+    def __init__(
+        self,
+        *,
+        reference: Iterable[str],
+        extractor: Callable[[str | dict[str, Any]], set[str]] | None = None,
+    ) -> None:
+        ref = {r for r in reference if r}
+        if not ref:
+            raise ValueError("Coverage requires a non-empty reference set")
+        self._reference: frozenset[str] = frozenset(ref)
+        self._extractor = extractor
+
+    async def evaluate(self, finding: Any, context: dict[str, Any]) -> EvalResult:
+        del context
+        output = finding.output if hasattr(finding, "output") else finding
+        found_normalised, raw_found = self._find_present(output)
+
+        matched = sorted(found_normalised)
+        missing = sorted(r for r in self._reference if r.lower() not in found_normalised)
+        score = (len(self._reference) - len(missing)) / len(self._reference)
+        label = "pass" if not missing else "warn" if matched else "fail"
+
+        return EvalResult(
+            evaluator=self.name,
+            score=score,
+            label=label,
+            reasoning=(
+                f"matched {len(matched)}/{len(self._reference)}; missing={missing}"
+                if missing
+                else f"matched {len(matched)}/{len(self._reference)}; all present"
+            ),
+            raw={
+                "matched": matched,
+                "missing": missing,
+                "extracted": sorted(raw_found),
+            },
+        )
+
+    def _find_present(self, output: Any) -> tuple[set[str], set[str]]:
+        """Return `(matched_normalised, raw_extracted)`.
+
+        `matched_normalised` is the subset of `self._reference`, lower-
+        cased, that the output contains. `raw_extracted` is whatever the
+        extractor produced (for diagnostics in `raw`).
+        """
+        if self._extractor is not None:
+            raw_found = self._extractor(output)
+            found_lower = {item.lower() for item in raw_found}
+            matched = {r.lower() for r in self._reference if r.lower() in found_lower}
+            return matched, raw_found
+
+        # Default substring match against a single text blob.
+        text = output if isinstance(output, str) else str(output)
+        text_lower = text.lower()
+        matched = {r.lower() for r in self._reference if r.lower() in text_lower}
+        return matched, {text}
+
+
+__all__ = ["Coverage"]

agentforge/eval/format_compliance.py

@@ -0,0 +1,107 @@
+"""`FormatCompliance` — deterministic grader for output shape.
+
+Three modes pick the constraint applied to `RunResult.output`:
+  - `regex=<pattern>` — output (str) must match the pattern.
+  - `pydantic_model=<BaseModel subclass>` — output validates against
+    the model. Accepts dict output directly or a string that
+    parses as JSON.
+  - `json_parseable=True` — output (str) must parse as JSON. No
+    schema enforcement.
+
+Exactly one mode must be set at construction. Future modes (Lark
+grammars, ANTLR, JSON Schema via the `jsonschema` dep) can land in
+follow-ups without breaking the constructor.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, ClassVar
+
+from agentforge_core.contracts.evaluator import EvalResult, Evaluator
+from pydantic import BaseModel, ValidationError
+
+
+class FormatCompliance(Evaluator):
+    """Score = 1.0 (passes) or 0.0 (fails). Label is `"pass"` / `"fail"`."""
+
+    name: ClassVar[str] = "format_compliance"
+    cost_estimate_usd: ClassVar[float] = 0.0
+
+    def __init__(
+        self,
+        *,
+        regex: str | None = None,
+        pydantic_model: type[BaseModel] | None = None,
+        json_parseable: bool = False,
+    ) -> None:
+        modes_set = sum(
+            1 for x in (regex is not None, pydantic_model is not None, json_parseable) if x
+        )
+        if modes_set != 1:
+            raise ValueError(
+                "FormatCompliance requires exactly one of regex=, pydantic_model=, json_parseable="
+            )
+        self._regex = re.compile(regex) if regex is not None else None
+        self._model = pydantic_model
+        self._json_parseable = json_parseable
+
+    async def evaluate(self, finding: Any, context: dict[str, Any]) -> EvalResult:
+        del context
+        output = finding.output if hasattr(finding, "output") else finding
+        if self._regex is not None:
+            return self._check_regex(output)
+        if self._model is not None:
+            return self._check_model(output)
+        return self._check_json(output)
+
+    def _check_regex(self, output: Any) -> EvalResult:
+        assert self._regex is not None
+        if not isinstance(output, str):
+            return _fail(f"regex mode requires string output; got {type(output).__name__}")
+        if self._regex.fullmatch(output):
+            return _pass(f"output matched regex {self._regex.pattern!r}")
+        return _fail(f"output did not match regex {self._regex.pattern!r}")
+
+    def _check_model(self, output: Any) -> EvalResult:
+        assert self._model is not None
+        candidate: Any
+        if isinstance(output, dict):
+            candidate = output
+        elif isinstance(output, str):
+            try:
+                candidate = json.loads(output)
+            except json.JSONDecodeError as exc:
+                return _fail(f"output is not JSON-parseable: {exc.msg}")
+        else:
+            return _fail(
+                f"pydantic_model mode requires dict or JSON string; got {type(output).__name__}"
+            )
+        try:
+            self._model.model_validate(candidate)
+        except ValidationError as exc:
+            return _fail(f"validation failed: {exc.errors(include_url=False)}")
+        return _pass(f"output validates against {self._model.__name__}")
+
+    def _check_json(self, output: Any) -> EvalResult:
+        if isinstance(output, dict):
+            return _pass("output is already a dict (JSON-compatible)")
+        if not isinstance(output, str):
+            return _fail(f"json_parseable mode requires str or dict; got {type(output).__name__}")
+        try:
+            json.loads(output)
+        except json.JSONDecodeError as exc:
+            return _fail(f"output is not JSON-parseable: {exc.msg}")
+        return _pass("output parses as JSON")
+
+
+def _pass(reasoning: str) -> EvalResult:
+    return EvalResult(evaluator=FormatCompliance.name, score=1.0, label="pass", reasoning=reasoning)
+
+
+def _fail(reasoning: str) -> EvalResult:
+    return EvalResult(evaluator=FormatCompliance.name, score=0.0, label="fail", reasoning=reasoning)
+
+
+__all__ = ["FormatCompliance"]