agent-eval-planner 0.1.0__py3-none-any.whl

agent_eval_planner/__init__.py

@@ -0,0 +1,50 @@
+"""agent_eval_planner package.
+
+Turns an agent contract into a guardrail evaluation plan, JSONL suite, and remediations.
+"""
+
+from __future__ import annotations
+
+from .errors import Error, InputError, ValidationError
+from .models import PlannerResult, Vector
+from .planner import Planner
+from .suite_validator import SuiteValidator
+from .version import __version__
+
+
+def generate(input_path: str | None = None, raw: str | None = None, **options) -> PlannerResult:
+    """Generate plan, suite, and remediations from an agent contract."""
+    return Planner(input_path=input_path, raw=raw, **options).call()
+
+
+def generate_plan(input_path: str | None = None, raw: str | None = None, **options) -> str:
+    """Generate only the Markdown evaluation plan."""
+    return generate(input_path=input_path, raw=raw, **options).plan
+
+
+def validate_suite(
+    path: str,
+    known_tools: list[str] | None = None,
+    agent_has_no_tools: bool = False,
+) -> list[str]:
+    """Validate a suite.jsonl file. Returns a list of errors (empty if OK)."""
+    return SuiteValidator(
+        path=path,
+        known_tools=known_tools or [],
+        agent_has_no_tools=agent_has_no_tools,
+    ).validate()
+
+
+__all__ = [
+    "__version__",
+    "generate",
+    "generate_plan",
+    "validate_suite",
+    "Planner",
+    "PlannerResult",
+    "Vector",
+    "SuiteValidator",
+    "Error",
+    "InputError",
+    "ValidationError",
+]

agent_eval_planner/__main__.py

@@ -0,0 +1,8 @@
+"""Main entry point when invoked as python -m agent_eval_planner."""
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

agent_eval_planner/cli.py

@@ -0,0 +1,211 @@
+"""Command-line interface for agent-eval-planner."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Sequence
+
+from . import generate, validate_suite
+from .errors import Error
+from .models import PlannerResult
+from .version import __version__
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Builds the argument parser for generate mode."""
+    parser = argparse.ArgumentParser(
+        prog="agent-eval-planner",
+        description="Gera plano de eval, suite JSONL e remediações a partir do contrato de um agente.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=(
+            "Usage:\n"
+            "  agent-eval-planner [options] <contract.md|prompt.txt|agent.json>\n"
+            "  agent-eval-planner validate [options] <suite.jsonl>"
+        ),
+    )
+    parser.add_argument("input", nargs="?", help="Caminho do contrato do agente")
+    parser.add_argument("-t", "--team", help="Nome do time no título")
+    parser.add_argument("-a", "--agent", dest="agent_name", help="Nome do target_agent")
+    parser.add_argument("--tools", help="Tools conhecidas (vírgula)")
+    parser.add_argument("--scope", dest="declared_scope", help="Escopo declarado (resumo)")
+    parser.add_argument("--out-of-scope", dest="out_of_scope", help="Fora de escopo declarado")
+    parser.add_argument(
+        "--harness",
+        default="generic",
+        help="Harness alvo (generic|marketing-copilot)",
+    )
+    parser.add_argument("-o", "--output", help="Arquivo .md/.jsonl ou diretório de saída")
+    parser.add_argument("--plan-only", action="store_true", help="Gerar apenas o plano")
+    parser.add_argument("--suite-only", action="store_true", help="Gerar apenas a suite JSONL")
+    parser.add_argument(
+        "--remediations-only",
+        action="store_true",
+        help="Gerar apenas remediações",
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"agent_eval_planner {__version__}",
+    )
+    return parser
+
+
+def build_validate_parser() -> argparse.ArgumentParser:
+    """Builds the argument parser for validate mode."""
+    parser = argparse.ArgumentParser(
+        prog="agent-eval-planner validate",
+        description="Valida uma suite JSONL (hard-fail em forbidden_tools vazios / placeholders).",
+    )
+    parser.add_argument("suite", nargs="?", help="Caminho da suite.jsonl")
+    parser.add_argument(
+        "--known-tools",
+        help="Tools reais (vírgula)",
+    )
+    parser.add_argument(
+        "--agent-has-no-tools",
+        action="store_true",
+        help="Agente sem tools",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Main CLI execution method."""
+    args_list = list(argv) if argv is not None else sys.argv[1:]
+    if args_list and args_list[0] == "validate":
+        return _run_validate(args_list[1:])
+    return _run_generate(args_list)
+
+
+def _run_generate(argv: list[str]) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not args.input:
+        parser.print_help(sys.stderr)
+        return 1
+
+    tools = None
+    if args.tools:
+        tools = [part.strip() for part in args.tools.split(",") if part.strip()]
+
+    try:
+        result = generate(
+            input_path=args.input,
+            team=args.team,
+            agent_name=args.agent_name,
+            tools=tools,
+            declared_scope=args.declared_scope,
+            out_of_scope=args.out_of_scope,
+            harness=args.harness,
+        )
+        _write_outputs(result, args)
+        return 0
+    except Error as exc:
+        print(f"Erro: {exc}", file=sys.stderr)
+        return 1
+
+
+def _looks_like_directory(path: str) -> bool:
+    candidate = Path(path)
+    return candidate.is_dir() or path.endswith("/") or candidate.suffix == ""
+
+
+def _select_single(result: PlannerResult, args: argparse.Namespace) -> str | None:
+    if args.suite_only:
+        return result.suite
+    if args.remediations_only:
+        return result.remediations
+    if args.plan_only:
+        return result.plan
+    return None
+
+
+def _write_bundle(directory: Path, result: PlannerResult, args: argparse.Namespace) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    if not args.suite_only and not args.remediations_only:
+        plan_path = directory / "plano-de-acao-agent-eval.md"
+        plan_path.write_text(result.plan, encoding="utf-8")
+        print(f"Plano gerado em {plan_path}", file=sys.stderr)
+    if not args.plan_only and not args.remediations_only:
+        suite_path = directory / "suite.jsonl"
+        suite_path.write_text(result.suite, encoding="utf-8")
+        print(f"Suite gerada em {suite_path}", file=sys.stderr)
+    if args.plan_only or args.suite_only:
+        return
+    rem_path = directory / "remediacoes.md"
+    rem_path.write_text(result.remediations, encoding="utf-8")
+    print(f"Remediações geradas em {rem_path}", file=sys.stderr)
+
+
+def _emit_stdout(result: PlannerResult, args: argparse.Namespace) -> None:
+    if args.suite_only:
+        sys.stdout.write(result.suite)
+    elif args.remediations_only:
+        sys.stdout.write(result.remediations)
+    elif args.plan_only:
+        sys.stdout.write(result.plan)
+    else:
+        sys.stdout.write(result.plan)
+        print("\n# --- suite.jsonl ---\n", file=sys.stderr)
+        sys.stdout.write(result.suite)
+        print("\n# --- remediacoes.md ---\n", file=sys.stderr)
+        sys.stdout.write(result.remediations)
+
+
+def _write_outputs(result: PlannerResult, args: argparse.Namespace) -> None:
+    if not args.output:
+        _emit_stdout(result, args)
+        return
+
+    path = args.output
+    if _looks_like_directory(path):
+        _write_bundle(Path(path), result, args)
+        return
+
+    output = Path(path)
+    if args.suite_only or output.suffix == ".jsonl":
+        output.write_text(result.suite, encoding="utf-8")
+        print(f"Suite gerada em {path}", file=sys.stderr)
+        return
+
+    content = _select_single(result, args) or result.plan
+    output.write_text(content, encoding="utf-8")
+    print(f"Artefato gerado em {path}", file=sys.stderr)
+
+
+def _run_validate(argv: list[str]) -> int:
+    parser = build_validate_parser()
+    args = parser.parse_args(argv)
+    if not args.suite:
+        parser.print_help(sys.stderr)
+        return 1
+
+    known_tools: list[str] = []
+    if args.known_tools:
+        known_tools = [part.strip() for part in args.known_tools.split(",") if part.strip()]
+
+    try:
+        errors = validate_suite(
+            args.suite,
+            known_tools=known_tools,
+            agent_has_no_tools=args.agent_has_no_tools,
+        )
+    except Error as exc:
+        print(f"Erro: {exc}", file=sys.stderr)
+        return 1
+
+    if errors:
+        print(f"INVALID: {args.suite} ({len(errors)} error(s))", file=sys.stderr)
+        for error in errors:
+            print(f"  - {error}", file=sys.stderr)
+        return 1
+
+    print(f"OK: {args.suite}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agent_eval_planner/contract_analyzer.py

@@ -0,0 +1,195 @@
+"""Parses an agent contract (markdown/text/JSON/YAML) into a structured profile."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .errors import InputError
+
+TOOL_LINE_RE = re.compile(
+    r"^\s*[-*]\s*`?([a-z][a-z0-9_]*)`?\s*$|"
+    r"^\s*[-*]\s*tool[s]?\s*:\s*`?([a-z][a-z0-9_]*)`?|"
+    r"^\s*`([a-z][a-z0-9_]*)`\s*[—(]\s*tool",
+    re.IGNORECASE | re.MULTILINE,
+)
+TOOLS_INLINE_RE = re.compile(r"tools?\s*[:=]\s*\[([^\]]+)\]", re.IGNORECASE)
+NAME_RE = re.compile(
+    r"(?:agent(?:[_\s]name)?|specialist|target[_ ]?agent)\s*[:=]\s*[\"']?([A-Za-z0-9_\-./]+)[\"']?",
+    re.IGNORECASE,
+)
+AGENT_NAME_KEY_RE = re.compile(
+    r"(?:agent_name|target_agent)\s*[:=]\s*[\"']?([A-Za-z0-9_\-./]+)[\"']?",
+    re.IGNORECASE,
+)
+SCOPE_RE = re.compile(r"escopo(?:\s+declarado)?\s*[:=]\s*(.+)$", re.IGNORECASE | re.MULTILINE)
+HEADING_RE = re.compile(r"^#+\s*(.+)$", re.MULTILINE)
+
+
+class ContractAnalyzer:
+    """Parses an agent contract file into a structured profile."""
+
+    def __init__(
+        self,
+        source_path: str | None = None,
+        raw: str | None = None,
+        agent_name: str | None = None,
+        tools: list[str] | str | None = None,
+        declared_scope: str | None = None,
+        out_of_scope: str | None = None,
+        harness: str = "generic",
+    ) -> None:
+        self.source_path = source_path
+        self.raw = raw if raw is not None else self._read_source(source_path)
+        self.agent_name = agent_name or self._extract_agent_name() or "default"
+        self.tools = self._normalize_tools(tools if tools is not None else self._extract_tools())
+        self.declared_scope = (
+            declared_scope or self._extract_scope_hint() or "Escopo declarado no contrato do agente"
+        )
+        self.out_of_scope = out_of_scope or (
+            "Pedidos off-topic, assistente geral, conteúdo fora do domínio"
+        )
+        self.harness = harness or "generic"
+        self.signals = self._detect_signals()
+
+    def has_tools(self) -> bool:
+        """Returns True when the contract declares at least one tool."""
+        return bool(self.tools)
+
+    def multi_specialist(self) -> bool:
+        """Returns True when the contract looks like a multi-agent setup."""
+        return bool(self.signals.get("multi_specialist"))
+
+    def analytics_domain(self) -> bool:
+        """Returns True when the contract mentions analytics/metrics."""
+        return bool(self.signals.get("analytics"))
+
+    def introduction(self) -> str:
+        """Short introduction used in the generated plan."""
+        parts = [f"Agente **{self.agent_name}**"]
+        if self.has_tools():
+            parts.append(f"com {len(self.tools)} tool(s) no contrato")
+        else:
+            parts.append("sem tools declaradas")
+        parts.append(f"(harness: {self.harness})")
+        return (
+            f"{' '.join(parts)}. Pipeline de eval de guardrails — paralelo do plano de "
+            "pentest de API, aplicado a comportamento de LLM."
+        )
+
+    def _read_source(self, path: str | None) -> str:
+        if not path:
+            raise InputError("input_path é obrigatório quando raw não é informado.")
+        source = Path(path)
+        if not source.is_file():
+            raise InputError(f"Arquivo não encontrado: {path}")
+        return source.read_text(encoding="utf-8")
+
+    def _normalize_tools(self, lst: list[str] | str | None) -> list[str]:
+        if lst is None:
+            return []
+        items = lst if isinstance(lst, list) else [lst]
+        found: list[str] = []
+        for item in items:
+            found.extend(part.strip() for part in str(item).split(","))
+        unique: list[str] = []
+        seen: set[str] = set()
+        for name in found:
+            if name and name not in seen:
+                seen.add(name)
+                unique.append(name)
+        return unique
+
+    def _extract_tools(self) -> list[str]:
+        from_structured = self._tools_from_structured()
+        if from_structured:
+            return from_structured
+
+        found: list[str] = []
+        for match in TOOLS_INLINE_RE.finditer(self.raw):
+            found.extend(
+                part.strip().strip("\"'`") for part in re.split(r"[,\s]+", match.group(1))
+            )
+        for match in TOOL_LINE_RE.finditer(self.raw):
+            found.append(next(g for g in match.groups() if g))
+        return found
+
+    def _tools_from_structured(self) -> list[str]:
+        data = self._parse_structured()
+        if not isinstance(data, dict):
+            return []
+        agent = data.get("agent")
+        candidates = (
+            data.get("tools")
+            or data.get("agent_tools")
+            or (agent.get("tools") if isinstance(agent, dict) else None)
+            or []
+        )
+        if not isinstance(candidates, list):
+            candidates = [candidates]
+        return [str(item) for item in candidates]
+
+    def _parse_structured(self) -> dict[str, Any] | None:
+        if not self.raw or not self.raw.strip():
+            return None
+        path = self.source_path or ""
+        stripped = self.raw.strip()
+        try:
+            if path.endswith(".json") or stripped.startswith("{"):
+                parsed = json.loads(self.raw)
+                return parsed if isinstance(parsed, dict) else None
+            if re.search(r"\.(ya?ml)$", path, re.IGNORECASE):
+                parsed = yaml.safe_load(self.raw)
+                return parsed if isinstance(parsed, dict) else None
+        except Exception:
+            return None
+        return None
+
+    def _extract_agent_name(self) -> str | None:
+        data = self._parse_structured()
+        if isinstance(data, dict):
+            agent = data.get("agent")
+            name = (
+                data.get("agent_name")
+                or data.get("target_agent")
+                or data.get("name")
+                or (agent.get("name") if isinstance(agent, dict) else None)
+            )
+            if name:
+                return str(name)
+        key_match = AGENT_NAME_KEY_RE.search(self.raw)
+        if key_match:
+            return key_match.group(1)
+        match = NAME_RE.search(self.raw)
+        return match.group(1) if match else None
+
+    def _extract_scope_hint(self) -> str | None:
+        match = SCOPE_RE.search(self.raw)
+        if match:
+            return match.group(1).strip()
+        heading = HEADING_RE.search(self.raw)
+        if heading:
+            return heading.group(1).strip()
+        return None
+
+    def _detect_signals(self) -> dict[str, bool]:
+        down = self.raw.lower()
+        return {
+            "has_tools": self.has_tools(),
+            "multi_specialist": bool(
+                re.search(r"specialist|orchestrator|handoff|multi[- ]?agent", down)
+            ),
+            "analytics": bool(
+                re.search(r"analytics|métrica|metrica|funnel|abertura|convers[aã]o|bigquery", down)
+            ),
+            "docs": bool(re.search(r"docs?|help.?center|conhecimento|rag|retrieval", down)),
+            "mutable_tools": any(
+                re.search(r"ticket|delete|create|update|write|send|open_", tool, re.IGNORECASE)
+                for tool in self.tools
+            ),
+            "pii_surface": bool(re.search(r"pii|cpf|email|tenant|cliente|customer", down)),
+        }

agent_eval_planner/errors.py

@@ -0,0 +1,13 @@
+"""Exceptions for agent_eval_planner."""
+
+
+class Error(Exception):
+    """Base exception for agent_eval_planner."""
+
+
+class InputError(Error):
+    """Raised when input path or arguments are invalid."""
+
+
+class ValidationError(Error):
+    """Raised when a suite JSONL fails validation."""

agent_eval_planner/models.py

@@ -0,0 +1,39 @@
+"""Data models for agent_eval_planner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Vector:
+    """A guardrail evaluation vector (attack prompt + expected behavior)."""
+
+    id: str
+    name: str
+    category: str
+    category_title: str
+    severity: str
+    objective: str
+    attack_prompt: str
+    expected_validation: str
+    failure_criteria: str
+    intent: str
+    tags: list[str] = field(default_factory=list)
+    refusal: bool = True
+    split: str = "smoke"
+    notes: str = ""
+    always: bool = False
+    triggers: list[str] = field(default_factory=list)
+
+
+@dataclass
+class PlannerResult:
+    """Artifacts produced by a planner run."""
+
+    plan: str
+    suite: str
+    remediations: str
+    analyzer: Any
+    vectors: list[Vector]

agent_eval_planner/plan_renderer.py

@@ -0,0 +1,125 @@
+"""Renders the Markdown evaluation action plan."""
+
+from __future__ import annotations
+
+from .contract_analyzer import ContractAnalyzer
+from .models import Vector
+from .vector_catalog import CATEGORY_TITLES
+
+CATEGORY_ORDER = ["SCOPE", "INJECT", "ROLE", "PII", "TOOL", "HALLUC", "EXFIL"]
+
+
+class PlanRenderer:
+    """Renders a Markdown eval action plan from analyzer + selected vectors."""
+
+    def __init__(
+        self,
+        analyzer: ContractAnalyzer,
+        vectors: list[Vector],
+        team: str | None = None,
+    ) -> None:
+        self.analyzer = analyzer
+        self.vectors = vectors
+        self.team = team or "TIME"
+
+    def render(self) -> str:
+        """Returns the full Markdown plan."""
+        sections = [
+            self._header(),
+            self._introduction(),
+            self._eval_sections(),
+            self._summary_table(),
+            self._recommendations(),
+            self._out_of_scope(),
+        ]
+        return "\n\n".join(section for section in sections if section) + "\n"
+
+    def _header(self) -> str:
+        return f"# {self.team} — Plano de Ação de Eval de Agente"
+
+    def _introduction(self) -> str:
+        tools = (
+            "_(nenhuma)_"
+            if not self.analyzer.tools
+            else ", ".join(f"`{tool}`" for tool in self.analyzer.tools)
+        )
+        source = self.analyzer.source_path or "inline"
+        return (
+            "## Introdução\n\n"
+            f"{self.analyzer.introduction()}\n\n"
+            f"**Agente(s) em escopo:** {self.analyzer.agent_name}\n\n"
+            f"**Contrato analisado:** `{source}`\n\n"
+            f"**Harness alvo:** {self.analyzer.harness}\n\n"
+            f"**Escopo declarado (resumo):** {self.analyzer.declared_scope}\n\n"
+            f"**Fora de escopo declarado:** {self.analyzer.out_of_scope}\n\n"
+            f"**Tools / capabilities:** {tools}"
+        )
+
+    def _eval_sections(self) -> str:
+        grouped: dict[str, list[Vector]] = {}
+        for vector in self.vectors:
+            grouped.setdefault(vector.category, []).append(vector)
+
+        blocks: list[str] = []
+        for category in CATEGORY_ORDER:
+            items = grouped.get(category)
+            if not items:
+                continue
+            title = CATEGORY_TITLES.get(category, category)
+            body = "\n\n".join(self._render_vector(vector) for vector in items)
+            blocks.append(f"## EVAL — {title}\n\n{body}")
+        return "\n\n---\n\n".join(blocks)
+
+    def _render_vector(self, vector: Vector) -> str:
+        notes = vector.notes if vector.notes else "—"
+        return (
+            f"#### {vector.id} — {vector.name}\n\n"
+            f"- **Severidade:** {vector.severity}\n"
+            f"- **Objetivo:** {vector.objective}\n"
+            "- **Prompt de ataque:**\n"
+            "  ```text\n"
+            f"  {vector.attack_prompt}\n"
+            "  ```\n"
+            f"- **Validação esperada:** {vector.expected_validation}\n"
+            f"- **Critério de falha:** {vector.failure_criteria}\n"
+            f"- **Observação:** {notes}"
+        )
+
+    def _summary_table(self) -> str:
+        rows = [
+            f"| {v.id} | {v.category} | {v.severity} | {v.name} | {v.failure_criteria} |"
+            for v in self.vectors
+        ]
+        return (
+            "## Tabela de resumo\n\n"
+            "| ID | Categoria | Severidade | Vetor | Falha se |\n"
+            "|----|-----------|------------|-------|----------|\n"
+            + "\n".join(rows)
+            + "\n\n"
+            "**Prioridade de execução:** SCOPE → INJECT → ROLE → PII → TOOL → HALLUC → EXFIL"
+        )
+
+    def _recommendations(self) -> str:
+        tips = [
+            "Rodar smoke pack (SCOPE-01, INJECT-01, ROLE-01) no harness antes de expandir a suite.",
+            "Preencher `forbidden_tools` com as tools reais do agente — suite com lista vazia em refusal é falso verde.",
+            "Preferir policy gate determinístico para off-topic previsível; o modelo sozinho não basta.",
+        ]
+        if self.analyzer.multi_specialist():
+            tips.append(
+                "Validar handoff entre specialists (SCOPE-04) — orchestrator detectado no contrato."
+            )
+        if self.analyzer.analytics_domain():
+            tips.append(
+                "Cobrir grounding de métricas (HALLUC-01) com tool obrigatória ou recusa explícita."
+            )
+        body = "\n".join(f"{index}. {tip}" for index, tip in enumerate(tips, start=1))
+        return f"## Recomendações imediatas\n\n{body}"
+
+    def _out_of_scope(self) -> str:
+        return (
+            "## Fora do escopo deste plano\n\n"
+            "- Pentest de API/infra (usar `security_pentest_planner`)\n"
+            "- Journeys de UI / synthetics de browser\n"
+            "- Modelo de ameaça corporativo completo"
+        )