agentforge-py 0.2.1__py3-none-any.whl

agentforge/eval/regression.py

@@ -0,0 +1,143 @@
+"""`RegressionVsBaseline` — deterministic grader against a locked baseline.
+
+The baseline file is a JSONL (one JSON object per line) where each
+entry is keyed by `task`:
+
+    {"task": "Summarise PR #42", "expected": "PR #42 adds X..."}
+    {"task": "List failing tests",  "expected": ["test_a", "test_b"]}
+
+At construction the file is loaded into a `{task: expected_output}`
+map. For each `evaluate` call the grader picks the baseline entry
+matching `context["task"]` and compares `finding.output` against it.
+
+Two modes pick the comparison:
+  - `mode="exact"` (default) — `output == expected`. Score 1.0 or 0.0.
+  - `mode="structural"` — output and expected are both dicts;
+    score = fraction of matching keys (case-sensitive on both keys
+    and values).
+
+Result labels:
+  - `"improved"` — exact-match in exact mode; perfect structural match.
+  - `"regressed"` — mismatch in either mode.
+  - `"no_baseline"` — no entry in the baseline file matches the task.
+    Score is NaN; the evaluator does not claim regression in this case.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+from typing import Any, ClassVar, Literal
+
+from agentforge_core.contracts.evaluator import EvalResult, Evaluator
+
+_Mode = Literal["exact", "structural"]
+
+
+class RegressionVsBaseline(Evaluator):
+    """Score the current run vs a locked baseline file."""
+
+    name: ClassVar[str] = "regression_vs_baseline"
+    cost_estimate_usd: ClassVar[float] = 0.0
+
+    def __init__(
+        self,
+        *,
+        baseline_path: str | Path,
+        mode: _Mode = "exact",
+    ) -> None:
+        if mode not in ("exact", "structural"):
+            raise ValueError(f"mode must be 'exact' or 'structural'; got {mode!r}")
+        self._mode = mode
+        self._baselines: dict[str, Any] = self._load(Path(baseline_path))
+
+    @staticmethod
+    def _load(path: Path) -> dict[str, Any]:
+        if not path.exists():
+            raise FileNotFoundError(f"baseline file not found: {path}")
+        out: dict[str, Any] = {}
+        with path.open(encoding="utf-8") as fh:
+            for lineno, raw in enumerate(fh, start=1):
+                line = raw.strip()
+                if not line:
+                    continue
+                try:
+                    entry = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise ValueError(
+                        f"baseline {path}:{lineno} is not valid JSON: {exc.msg}"
+                    ) from exc
+                if not isinstance(entry, dict) or "task" not in entry or "expected" not in entry:
+                    raise ValueError(
+                        f"baseline {path}:{lineno} must have 'task' and 'expected' keys"
+                    )
+                out[entry["task"]] = entry["expected"]
+        if not out:
+            raise ValueError(f"baseline file {path} is empty")
+        return out
+
+    async def evaluate(self, finding: Any, context: dict[str, Any]) -> EvalResult:
+        task = context.get("task")
+        if not isinstance(task, str) or task not in self._baselines:
+            return EvalResult(
+                evaluator=self.name,
+                score=math.nan,
+                label="no_baseline",
+                reasoning=f"no baseline entry for task {task!r}",
+            )
+
+        expected = self._baselines[task]
+        output = finding.output if hasattr(finding, "output") else finding
+
+        if self._mode == "exact":
+            matches = output == expected
+            return EvalResult(
+                evaluator=self.name,
+                score=1.0 if matches else 0.0,
+                label="improved" if matches else "regressed",
+                reasoning="exact match" if matches else "output differs from baseline",
+                raw={"expected": expected, "actual": output},
+            )
+
+        # Structural mode.
+        if not isinstance(expected, dict) or not isinstance(output, dict):
+            return EvalResult(
+                evaluator=self.name,
+                score=0.0,
+                label="regressed",
+                reasoning=(
+                    f"structural mode requires dict output and dict baseline; "
+                    f"got output={type(output).__name__}, expected={type(expected).__name__}"
+                ),
+            )
+        return self._compare_structural(output, expected)
+
+    def _compare_structural(self, output: dict[str, Any], expected: dict[str, Any]) -> EvalResult:
+        all_keys = expected.keys() | output.keys()
+        matching = [
+            k for k in all_keys if k in expected and k in output and output[k] == expected[k]
+        ]
+        missing = sorted(k for k in expected if k not in output)
+        extra = sorted(k for k in output if k not in expected)
+        mismatched = sorted(k for k in expected if k in output and output[k] != expected[k])
+        score = len(matching) / len(all_keys) if all_keys else 1.0
+        label = "improved" if score == 1.0 else "regressed"
+        return EvalResult(
+            evaluator=self.name,
+            score=score,
+            label=label,
+            reasoning=(
+                f"structural match {len(matching)}/{len(all_keys)} keys "
+                f"(missing={missing}, extra={extra}, mismatched={mismatched})"
+            ),
+            raw={
+                "matched_keys": sorted(matching),
+                "missing_keys": missing,
+                "extra_keys": extra,
+                "mismatched_keys": mismatched,
+            },
+        )
+
+
+__all__ = ["RegressionVsBaseline"]

agentforge/findings.py

@@ -0,0 +1,166 @@
+"""Finding variants and helper value types (feat-008).
+
+The `Finding` Protocol itself lives in `agentforge_core.contracts.finding`
+(shipped under feat-001). This module ships the four built-in variants
+plus the `Patch` and `Span` helpers two of them embed.
+
+Variants are **frozen Pydantic v2 models** (per ADR-0014, deviating
+from spec §4.2's `@dataclass` sketch). The external shape is the same;
+the model framework gives us validation on construction, declarative
+schema, and `model_dump` / `model_validate` round-trip — important
+because findings cross persistence (`Claim.payload`), transport
+(A2A / MCP / pipeline aggregation), and LLM-output (structured-output
+emission) boundaries.
+
+Each variant satisfies the `Finding` Protocol structurally — no
+inheritance from the Protocol is required. The Protocol is
+`runtime_checkable`, so `isinstance(x, Finding)` works.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Self
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class Patch(BaseModel):
+    """A structured diff embedded inside a `PatchFinding`.
+
+    Attributes:
+        file: Path to the file the diff applies against, relative to
+            repo root (e.g. `"src/foo.py"`).
+        diff: Unified-diff text. Must be a complete, applyable hunk
+            (or set of hunks against a single file).
+        hunk_count: Number of `@@` headers in `diff`. Cached at
+            construction; renderers may use it for summary stats.
+    """
+
+    model_config = ConfigDict(frozen=True, strict=True)
+
+    file: str = Field(min_length=1)
+    diff: str = Field(min_length=1)
+    hunk_count: int = Field(default=1, ge=1)
+
+
+class Span(BaseModel):
+    """A single source-range citation inside a `MultiSpanFinding`.
+
+    Attributes:
+        file: Path the span is in, relative to repo root.
+        start_line: 1-indexed first line of the span.
+        end_line: 1-indexed last line of the span (inclusive). Must
+            be `>= start_line`.
+        excerpt: The text on those lines (optional; useful for
+            renderers that can't read the source file themselves,
+            e.g. when findings are persisted and rendered later).
+    """
+
+    model_config = ConfigDict(frozen=True, strict=True)
+
+    file: str = Field(min_length=1)
+    start_line: int = Field(ge=1)
+    end_line: int = Field(ge=1)
+    excerpt: str = ""
+
+    @field_validator("end_line")
+    @classmethod
+    def _end_after_start(cls, end_line: int, info: Any) -> int:
+        start = info.data.get("start_line")
+        if start is not None and end_line < start:
+            raise ValueError(f"end_line ({end_line}) must be >= start_line ({start})")
+        return end_line
+
+
+class _FindingBase(BaseModel):
+    """Internal base — shared config + `to_dict` / `from_dict` plumbing.
+
+    Not part of the public surface. Each variant subclasses this to
+    inherit the round-trip helpers. Subclasses MUST declare the
+    Protocol-required attributes (`severity`, `category`, `message`)
+    themselves so `isinstance(x, Finding)` resolves correctly.
+    """
+
+    model_config = ConfigDict(frozen=True, strict=True)
+
+    def to_dict(self) -> dict[str, Any]:
+        """JSON-compatible serialisation. Round-trips via `from_dict`."""
+        return self.model_dump(mode="json")
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Self:
+        """Reconstruct a typed variant from a `to_dict()` payload."""
+        return cls.model_validate(data)
+
+
+class SimpleFinding(_FindingBase):
+    """The default variant — a severity-tagged issue or observation.
+
+    Use for code-review-style output, lint hits, audit notes — anything
+    that maps to "one finding = one location + one recommendation".
+    """
+
+    severity: str = Field(min_length=1)
+    category: str = Field(min_length=1)
+    message: str = Field(min_length=1)
+    recommendation: str = ""
+    file: str = ""
+    line: int | None = None
+    rule_id: str = ""
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class PatchFinding(_FindingBase):
+    """Finding that ships a structured patch the consumer can apply.
+
+    Use for refactor bots, codemod agents, and auto-fix suggestions.
+    `confidence` is a model-supplied estimate in `[0, 1]`; downstream
+    automation typically gates application on it.
+    """
+
+    severity: str = Field(min_length=1)
+    category: str = Field(min_length=1)
+    message: str = Field(min_length=1)
+    patch: Patch
+    rationale: str = Field(min_length=1)
+    confidence: float = Field(ge=0.0, le=1.0)
+
+
+class NarrativeFinding(_FindingBase):
+    """Long-form prose answer with citations.
+
+    Use for docs-Q&A, research summaries, explanatory output. `body`
+    is markdown; `references` is a flat list of pointer strings (free
+    form — typical shapes are `"path:line"`, URLs, or section anchors).
+    """
+
+    severity: str = Field(min_length=1)
+    category: str = Field(min_length=1)
+    message: str = Field(min_length=1)
+    body: str = Field(min_length=1)
+    references: list[str] = Field(default_factory=list)
+
+
+class MultiSpanFinding(_FindingBase):
+    """One logical issue manifested across multiple source locations.
+
+    Use for cross-file findings like "hard-coded secret present in N
+    files" or "this deprecated API is used in N places". `spans` lists
+    every site; a renderer typically produces one block per span.
+    """
+
+    severity: str = Field(min_length=1)
+    category: str = Field(min_length=1)
+    message: str = Field(min_length=1)
+    spans: list[Span] = Field(min_length=1)
+    recommendation: str = ""
+
+
+__all__ = [
+    "MultiSpanFinding",
+    "NarrativeFinding",
+    "Patch",
+    "PatchFinding",
+    "SimpleFinding",
+    "Span",
+]

agentforge/guardrails/__init__.py

@@ -0,0 +1,32 @@
+"""Built-in basic guardrail validators (feat-018).
+
+Four validators ship in the runtime package as the default tier:
+
+- `PromptInjectionBasic` (`prompt_injection_basic`) — regex-based
+  pattern matching for the common prompt-injection phrases.
+- `PIIRedactBasic` (`pii_redact_basic`) — regex-based PII
+  detection + redaction (email / phone / SSN / credit-card /
+  IPv4). Output validator; sets `redacted_content`.
+- `CapabilityCheck` (`capability_check`) — denies tools that
+  declare a `destructive` capability unless explicitly
+  allowlisted.
+- `Allowlist` (`allowlist`) — bare-name allowlist for tool gates.
+
+All four register themselves with the global Resolver under the
+matching category (`guardrails.input` / `guardrails.output` /
+`guardrails.tool_gates`) at import time.
+"""
+
+from __future__ import annotations
+
+from agentforge.guardrails.allowlist import Allowlist
+from agentforge.guardrails.capability_check import CapabilityCheck
+from agentforge.guardrails.pii_redact_basic import PIIRedactBasic
+from agentforge.guardrails.prompt_injection_basic import PromptInjectionBasic
+
+__all__ = [
+    "Allowlist",
+    "CapabilityCheck",
+    "PIIRedactBasic",
+    "PromptInjectionBasic",
+]

agentforge/guardrails/allowlist.py

@@ -0,0 +1,49 @@
+"""`allowlist` — bare-name allowlist for tool dispatch (feat-018).
+
+If `allowed` is set, only the tools whose names appear in it can
+run. Pairs naturally with `capability_check` to add a second
+restrictive layer.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from agentforge_core.contracts.guardrails import ToolCallGate
+from agentforge_core.contracts.tool import Tool
+from agentforge_core.resolver import register
+from agentforge_core.values.guardrails import ValidationResult
+
+
+@register("guardrails.tool_gates", "allowlist")
+class Allowlist(ToolCallGate):
+    """Only tools whose name appears in `allowed` can run."""
+
+    name: ClassVar[str] = "allowlist"
+    description: ClassVar[str] = (
+        "Permits only tools whose names appear in `allowed`. Everything else is blocked."
+    )
+    cost_estimate_ms: ClassVar[int] = 0
+
+    def __init__(self, *, allowed: list[str] | None = None) -> None:
+        self._allowed = set(allowed or ())
+
+    async def authorize(
+        self,
+        tool_name: str,
+        tool: Tool,
+        args: dict[str, Any],
+        context: dict[str, Any],
+    ) -> ValidationResult:
+        del tool, args, context
+        if tool_name in self._allowed:
+            return ValidationResult.ok()
+        return ValidationResult(
+            passed=False,
+            score=0.0,
+            violations=("not_in_allowlist",),
+            metadata={"tool": tool_name, "allowed": sorted(self._allowed)},
+        )
+
+
+__all__ = ["Allowlist"]

agentforge/guardrails/capability_check.py

@@ -0,0 +1,58 @@
+"""`capability_check` — denies tools tagged `destructive` unless
+explicitly allowlisted (feat-018).
+
+Tools declare their capabilities via the `Tool.capabilities`
+ClassVar (feat-004). Any tool that includes `"destructive"` in
+its capability set is blocked by this gate by default; the
+agent's config can opt-in specific tools via the
+`destructive_allow` list.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from agentforge_core.contracts.guardrails import ToolCallGate
+from agentforge_core.contracts.tool import Tool
+from agentforge_core.resolver import register
+from agentforge_core.values.guardrails import ValidationResult
+
+_DESTRUCTIVE = "destructive"
+
+
+@register("guardrails.tool_gates", "capability_check")
+class CapabilityCheck(ToolCallGate):
+    """Block `destructive` tools unless explicitly allowlisted."""
+
+    name: ClassVar[str] = "capability_check"
+    description: ClassVar[str] = (
+        "Denies any tool whose capabilities include 'destructive' unless "
+        "the tool name appears in `destructive_allow`."
+    )
+    cost_estimate_ms: ClassVar[int] = 0
+
+    def __init__(self, *, destructive_allow: list[str] | None = None) -> None:
+        self._allow = set(destructive_allow or ())
+
+    async def authorize(
+        self,
+        tool_name: str,
+        tool: Tool,
+        args: dict[str, Any],
+        context: dict[str, Any],
+    ) -> ValidationResult:
+        del args, context
+        caps = set(type(tool).capabilities)
+        if _DESTRUCTIVE not in caps:
+            return ValidationResult.ok()
+        if tool_name in self._allow:
+            return ValidationResult.ok()
+        return ValidationResult(
+            passed=False,
+            score=0.0,
+            violations=("destructive_not_allowlisted",),
+            metadata={"tool": tool_name, "capabilities": sorted(caps)},
+        )
+
+
+__all__ = ["CapabilityCheck"]