agentjury 0.4.4__py3-none-any.whl

agentjury/__init__.py

@@ -0,0 +1,22 @@
+"""AgentJury: peer review for AI agents."""
+
+from .aggregate import aggregate
+from .panel import Panel
+from .protocol import (
+    SCHEMA_VERSION,
+    Artifact,
+    Finding,
+    HumanReview,
+    Producer,
+    Review,
+    ReviewRequest,
+    Verdict,
+    Vote,
+)
+
+__version__ = "0.4.4"
+
+__all__ = [
+    "SCHEMA_VERSION", "Artifact", "Finding", "HumanReview", "Producer",
+    "Review", "ReviewRequest", "Verdict", "Vote", "Panel", "aggregate",
+]

agentjury/aggregate.py

@@ -0,0 +1,120 @@
+"""
+Turn a list of independent Reviews into one Verdict.
+
+No LLM calls here. The rules are written down so anyone can predict the verdict:
+
+  voters       responding judges who did not abstain
+  up / down    count of approve / revise votes among voters
+  score        mean of voters' scores
+  consensus    share of voters who sided with the majority vote
+  diversity    distinct providers among voters / voters
+  confidence   heuristic index: consensus, discounted for small panels, wide
+               score spread, and low diversity. NOT a calibrated probability.
+
+  status
+    insufficient_jury  fewer than `quorum` judges voted, OR the panel was built
+                       from two or more providers but only one provider's judges
+                       voted. Votes are reported; no verdict is reached.
+    blocked            blocking findings from at least two providers (or from at
+                       least two judges when the panel has only one provider).
+                       No single judge can block on its own.
+    needs_revision     majority revise, a tie, or exactly one blocking source.
+    verified           majority approve and no blocking finding.
+
+  quorum       default is a strict majority of requested judges:
+               1->1, 2->2, 3->2, 4->3, 5->3, 6->4
+
+Reviewer reputation will later weight these votes. For now every voter counts once.
+"""
+
+from __future__ import annotations
+
+from .protocol import Review, ReviewRequest, Verdict, Vote
+
+
+def default_quorum(requested: int) -> int:
+    """Strict majority of the requested panel."""
+    return max(1, requested // 2 + 1)
+
+
+def _empty(request: ReviewRequest, reviews, errors, requested, quorum, panel_id) -> Verdict:
+    return Verdict(
+        request_id=request.request_id, panel_id=panel_id,
+        requested=requested, responded=len(reviews), abstained=len(reviews), quorum=quorum,
+        task_type=request.task_type, domain=request.domain, producer=request.producer,
+        up=0, down=0, score=0.0, consensus=0.0, diversity=0.0, confidence=0.0,
+        status="insufficient_jury", reviews=reviews, errors=errors or [],
+    )
+
+
+def aggregate(
+    request: ReviewRequest,
+    reviews: list[Review],
+    errors: list[str] | None = None,
+    *,
+    requested: int | None = None,
+    quorum: int | None = None,
+    panel_id: str | None = None,
+    requested_providers: int | None = None,
+) -> Verdict:
+    requested = requested if requested is not None else len(reviews)
+    quorum = quorum if quorum is not None else default_quorum(requested)
+
+    voters = [r for r in reviews if r.vote != Vote.ABSTAIN]
+    abstained = len(reviews) - len(voters)
+    if not voters:
+        return _empty(request, reviews, errors, requested, quorum, panel_id)
+
+    n = len(voters)
+    up = sum(1 for r in voters if r.vote == Vote.APPROVE)
+    down = n - up
+    scores = [r.score for r in voters]
+    score = sum(scores) / n
+
+    consensus = max(up, down) / n
+    providers = {r.provider for r in voters}
+    diversity = len(providers) / n
+
+    panel_factor = n / (n + 1)  # 1 voter -> 0.5, 3 -> 0.75, 5 -> 0.83
+    spread_factor = 1 - (max(scores) - min(scores)) / 10  # identical scores -> 1.0
+    diversity_factor = 0.5 + 0.5 * diversity  # all one provider (n=3) -> 0.67; all distinct -> 1.0
+    confidence = round(consensus * panel_factor * spread_factor * diversity_factor, 3)
+
+    blocking_reviews = [r for r in voters if r.blocking]
+    blocking_providers = {r.provider for r in blocking_reviews}
+    independent_blocks = len(blocking_providers) >= 2 or (len(providers) == 1 and len(blocking_reviews) >= 2)
+
+    # A multi-provider panel that only heard from one provider has lost the
+    # independence it was built for, so it cannot reach a verdict.
+    wanted_providers = requested_providers if requested_providers is not None else len(providers)
+    provider_floor = min(2, wanted_providers)
+
+    if n < quorum or len(providers) < provider_floor:
+        status = "insufficient_jury"
+    elif independent_blocks:
+        status = "blocked"
+    elif blocking_reviews or up <= down:
+        status = "needs_revision"
+    else:
+        status = "verified"
+
+    return Verdict(
+        request_id=request.request_id,
+        panel_id=panel_id,
+        requested=requested,
+        responded=len(reviews),
+        abstained=abstained,
+        quorum=quorum,
+        task_type=request.task_type,
+        domain=request.domain,
+        producer=request.producer,
+        up=up,
+        down=down,
+        score=round(score, 2),
+        consensus=round(consensus, 3),
+        diversity=round(diversity, 3),
+        confidence=confidence,
+        status=status,
+        reviews=reviews,
+        errors=errors or [],
+    )

agentjury/cli.py

@@ -0,0 +1,312 @@
+"""
+Command-line interface.
+
+    agentjury review TASK OUTPUT [--panel SPEC] [--roles FILE] [--quorum N] [--task-type T] [--domain D] [--json]
+    agentjury roles [--roles FILE]
+    agentjury schema [request|verdict]
+    agentjury verdicts [--dir DIR] [-n N]
+    agentjury adjudicate ID [--judge J] [--finding N LABEL]... [--verdict agree|partial|disagree]
+                            [--producer-verdict correct|flawed] [--note TEXT] [--dir DIR]
+
+Verdicts are read from --dir, else $AGENTJURY_VERDICT_DIR, else .agentjury/verdicts.
+
+Exit codes: 0 verified, 1 needs_revision, 2 blocked, 3 insufficient_jury.
+
+TASK and OUTPUT are files (or "-" to read OUTPUT from stdin).
+PANEL is a comma-separated list of role:provider pairs, for example
+    accuracy:openai,critic:anthropic,executive:openai
+Every verdict is saved to .agentjury/verdicts/<request_id>.json so that
+reviews accumulate over time.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from .judges import ROLES, anthropic_judge, load_roles, openai_judge
+from .judges.base import Judge
+from .panel import Panel
+from .protocol import HumanReview, Producer, ReviewRequest, Verdict
+
+DEFAULT_PANEL = "accuracy:openai,critic:anthropic,executive:openai"
+VERDICT_DIR = Path(".agentjury") / "verdicts"
+
+PROVIDERS = {
+    "openai": openai_judge,
+    "anthropic": anthropic_judge,
+}
+
+SEVERITY_MARK = {"minor": "-", "major": "!", "blocking": "X"}
+
+
+# Exit codes, so shell scripts and CI can branch without parsing output.
+EXIT = {"verified": 0, "needs_revision": 1, "blocked": 2, "insufficient_jury": 3}
+
+
+def build_panel(spec: str, quorum: int | None = None) -> Panel:
+    judges: list[Judge] = []
+    for item in spec.split(","):
+        item = item.strip()
+        if not item:
+            continue
+        try:
+            role, provider = item.split(":")
+        except ValueError:
+            sys.exit(f"Bad panel entry {item!r}. Use role:provider, e.g. critic:anthropic")
+        if role not in ROLES:
+            sys.exit(f"Unknown role {role!r}. Known roles: {', '.join(sorted(ROLES))}")
+        if provider not in PROVIDERS:
+            sys.exit(f"Unknown provider {provider!r}. Known providers: {', '.join(PROVIDERS)}")
+        judges.append(PROVIDERS[provider](role))
+    return Panel(judges, quorum=quorum)
+
+
+def read(path: str) -> str:
+    if path == "-":
+        return sys.stdin.read()
+    return Path(path).read_text(encoding="utf-8")
+
+
+def save(verdict: Verdict) -> Path:
+    VERDICT_DIR.mkdir(parents=True, exist_ok=True)
+    out = VERDICT_DIR / verdict.filename
+    out.write_text(verdict.model_dump_json(indent=2), encoding="utf-8")
+    return out
+
+
+def print_verdict(verdict: Verdict) -> None:
+    print(verdict.render())
+    print(f"jury confidence index {verdict.confidence:.0%}  (heuristic, not a probability)")
+    if verdict.status == "insufficient_jury":
+        voters = verdict.responded - verdict.abstained
+        providers = len({r.provider for r in verdict.reviews if r.vote != "abstain"})
+        print(f"Insufficient jury: {voters} of {verdict.requested} judges voted (quorum {verdict.quorum}), "
+              f"from {providers} provider(s). No verdict.")
+    print()
+    for r in verdict.reviews:
+        arrow = {"approve": "▲", "revise": "▼", "abstain": "–"}[r.vote]
+        meta = f"{r.latency_ms / 1000:.1f}s" if r.latency_ms is not None else ""
+        print(f"{arrow} {r.score:>2.0f}  {r.judge:<22} {r.reason}  [{meta}]")
+        for f in r.findings:
+            print(f"        {SEVERITY_MARK[f.severity]} {f.text}")
+    for e in verdict.errors:
+        print(f"!  {e}")
+
+
+def cmd_review(args: argparse.Namespace) -> int:
+    if args.roles:
+        load_roles(args.roles)
+    request = ReviewRequest(
+        task=read(args.task),
+        output=read(args.output),
+        context=read(args.context) if args.context else None,
+        task_type=args.task_type,
+        domain=args.domain,
+        producer=Producer(
+            agent=args.agent,
+            framework=args.framework,
+            provider=args.producer_provider,
+            model=args.producer_model,
+        ),
+    )
+    verdict = build_panel(args.panel, quorum=args.quorum).review(request)
+
+    if args.json:
+        print(verdict.model_dump_json(indent=2))
+    else:
+        print_verdict(verdict)
+
+    if not args.no_save:
+        path = save(verdict)
+        if not args.json:
+            print(f"\nsaved {path}")
+
+    return EXIT[verdict.status]
+
+
+def verdict_dir(args: argparse.Namespace) -> Path:
+    return Path(getattr(args, "dir", None) or os.environ.get("AGENTJURY_VERDICT_DIR") or VERDICT_DIR)
+
+
+def load_verdict(args: argparse.Namespace) -> tuple[Path, Verdict]:
+    ref = args.request_id
+    path = Path(ref)
+    if not path.is_file():
+        d = verdict_dir(args)
+        candidates = sorted(f for f in d.glob("*.json") if ref in f.stem) if d.is_dir() else []
+        if len(candidates) != 1:
+            hint = f"{len(candidates)} matches" if candidates else "no match"
+            sys.exit(f"Cannot find verdict {ref!r} in {d} ({hint}). Use `agentjury verdicts --dir {d}` to list.")
+        path = candidates[0]
+    return path, Verdict.model_validate_json(path.read_text(encoding="utf-8"))
+
+
+def cmd_verdicts(args: argparse.Namespace) -> int:
+    d = verdict_dir(args)
+    files = sorted(d.glob("*.json"), key=lambda f: f.stat().st_mtime, reverse=True) if d.is_dir() else []
+    if not files:
+        print(f"No verdicts in {d}")
+        return 0
+    print(f"{d}\n")
+    for f in files[: args.n]:
+        v = Verdict.model_validate_json(f.read_text(encoding="utf-8"))
+        graded = sum(1 for r in v.reviews for fi in r.findings if fi.adjudication)
+        total = sum(len(r.findings) for r in v.reviews)
+        mark = f"  [adjudicated {graded}/{total}]" if graded else ""
+        prod = f"  producer:{v.human_verdict}" if v.human_verdict else ""
+        print(f"run {v.run_id}  req {v.request_id}  {v.created_at:%Y-%m-%d %H:%M}  {v.render()}{mark}{prod}")
+    return 0
+
+
+ADJUDICATION_LOG = "adjudications.jsonl"
+
+
+def _adjudicator() -> str:
+    import getpass
+    return os.environ.get("AGENTJURY_ADJUDICATOR") or getpass.getuser()
+
+
+def log_event(directory: Path, event: dict) -> None:
+    """Append-only audit trail. The verdict JSON holds current state; this holds history."""
+    from uuid import uuid4
+    event = {"event_id": uuid4().hex[:12], **event}
+    with (directory / ADJUDICATION_LOG).open("a", encoding="utf-8") as f:
+        f.write(json.dumps(event, default=str) + "\n")
+
+
+def cmd_adjudicate(args: argparse.Namespace) -> int:
+    from datetime import datetime, timezone
+
+    path, v = load_verdict(args)
+    now = datetime.now(timezone.utc)
+    who = _adjudicator()
+    base = {"at": now.isoformat(timespec="seconds"), "adjudicator": who,
+            "run_id": v.run_id, "request_id": v.request_id, "note": args.note}
+    changed: list[str] = []
+
+    if args.finding or args.verdict:
+        if not args.judge:
+            sys.exit("--judge is required when grading findings or a review. Judges: " +
+                     ", ".join(r.judge for r in v.reviews))
+        matches = [r for r in v.reviews if r.judge == args.judge or r.review_id == args.judge
+                   or r.role == args.judge]
+        if len(matches) != 1:
+            sys.exit(f"--judge {args.judge!r} matched {len(matches)} reviews. Judges: " +
+                     ", ".join(r.judge for r in v.reviews))
+        review = matches[0]
+        for ref, label in args.finding or []:
+            if label not in ("correct", "partially_correct", "wrong"):
+                sys.exit(f"Finding label must be correct, partially_correct, or wrong; got {label!r}.")
+            target = None
+            if ref.isdigit() and 1 <= int(ref) <= len(review.findings):
+                target = review.findings[int(ref) - 1]
+            else:
+                target = next((f for f in review.findings if f.id == ref), None)
+            if target is None:
+                sys.exit(f"{review.judge} has no finding {ref!r} (it has {len(review.findings)}).")
+            log_event(path.parent, {**base, "kind": "finding", "review_id": review.review_id,
+                                    "judge": review.judge, "config_id": review.config_id,
+                                    "finding_id": target.id, "old": target.adjudication, "new": label})
+            target.adjudication = label
+            target.adjudicated_at = now
+            changed.append(f"{review.judge} finding {ref}: {label}")
+        if args.verdict:
+            old = review.human_review.verdict if review.human_review else None
+            log_event(path.parent, {**base, "kind": "review", "review_id": review.review_id,
+                                    "judge": review.judge, "config_id": review.config_id,
+                                    "old": old, "new": args.verdict})
+            review.human_review = HumanReview(verdict=args.verdict, note=args.note, reviewed_at=now)
+            changed.append(f"{review.judge} review: {args.verdict}")
+
+    if args.producer_verdict:
+        log_event(path.parent, {**base, "kind": "producer", "old": v.human_verdict, "new": args.producer_verdict})
+        v.human_verdict = args.producer_verdict
+        v.human_note = args.note
+        v.adjudicated_at = now
+        changed.append(f"producer output: {args.producer_verdict}")
+
+    if not changed:
+        sys.exit("Nothing to record. Give --finding, --verdict, or --producer-verdict.")
+
+    path.write_text(v.model_dump_json(indent=2), encoding="utf-8")
+    print(f"run {v.run_id}  {path}")
+    for c in changed:
+        print(f"  {c}")
+    print(f"  logged by {who} -> {path.parent / ADJUDICATION_LOG}")
+    return 0
+
+
+def cmd_roles(args: argparse.Namespace) -> int:
+    if args.roles:
+        load_roles(args.roles)
+    for name, desc in ROLES.items():
+        print(f"{name:<10} {desc}")
+    return 0
+
+
+def cmd_schema(args: argparse.Namespace) -> int:
+    model = {"request": ReviewRequest, "verdict": Verdict}[args.object]
+    print(json.dumps(model.model_json_schema(), indent=2))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    load_dotenv()
+    parser = argparse.ArgumentParser(prog="agentjury", description="Peer review for AI agent output.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("review", help="Review an agent's output with a panel of judges.")
+    p.add_argument("task", help="File containing the task the agent was given.")
+    p.add_argument("output", help="File containing the agent's output, or - for stdin.")
+    p.add_argument("--context", help="File with background the judges should know.")
+    p.add_argument("--panel", default=os.environ.get("AGENTJURY_PANEL", DEFAULT_PANEL),
+                   help=f"role:provider pairs, comma-separated (default: {DEFAULT_PANEL})")
+    p.add_argument("--roles", default=os.environ.get("AGENTJURY_ROLES"),
+                   help="JSON file of extra roles {name: description}, e.g. a domain expert.")
+    p.add_argument("--quorum", type=int, help="Minimum judges that must respond (default: majority).")
+    p.add_argument("--task-type", help="Kind of work, e.g. financial_analysis, code_review, summary.")
+    p.add_argument("--domain", help="Subject area, e.g. private_credit, python.")
+    p.add_argument("--agent", help="Name of the agent that did the work.")
+    p.add_argument("--framework", help="Framework the agent runs on, e.g. hermes.")
+    p.add_argument("--producer-provider", help="Provider of the model that did the work, e.g. anthropic.")
+    p.add_argument("--producer-model", help="Model that did the work, e.g. claude-fable-5-1.")
+    p.add_argument("--json", action="store_true", help="Print the full verdict as JSON.")
+    p.add_argument("--no-save", action="store_true", help="Do not write the verdict to .agentjury/.")
+    p.set_defaults(func=cmd_review)
+
+    r = sub.add_parser("roles", help="List available judge roles.")
+    r.add_argument("--roles", default=os.environ.get("AGENTJURY_ROLES"), help="JSON file of extra roles.")
+    r.set_defaults(func=cmd_roles)
+
+    vl = sub.add_parser("verdicts", help="List saved verdicts, newest first.")
+    vl.add_argument("--dir", help="Verdict directory (default: $AGENTJURY_VERDICT_DIR or .agentjury/verdicts).")
+    vl.add_argument("-n", type=int, default=20, help="How many to show.")
+    vl.set_defaults(func=cmd_verdicts)
+
+    a = sub.add_parser("adjudicate", help="Record a human judgement on a saved verdict.")
+    a.add_argument("request_id", metavar="ID", help="run_id or request_id (or unique fragment), or a path to the JSON file.")
+    a.add_argument("--dir", help="Verdict directory (default: $AGENTJURY_VERDICT_DIR or .agentjury/verdicts).")
+    a.add_argument("--judge", help="Which review to grade: judge name (critic/anthropic), role, or review_id.")
+    a.add_argument("--finding", nargs=2, action="append", metavar=("N", "LABEL"),
+                   help="Grade finding N (1-based, as shown) as correct, partially_correct, or wrong. Repeatable.")
+    a.add_argument("--verdict", choices=["agree", "partial", "disagree"], help="Your overall view of that judge's review.")
+    a.add_argument("--producer-verdict", choices=["correct", "flawed"], help="Your view of the agent's output itself.")
+    a.add_argument("--note", help="Free-text reason, stored with the review and/or producer verdict.")
+    a.set_defaults(func=cmd_adjudicate)
+
+    s = sub.add_parser("schema", help="Print the JSON schema for the protocol objects.")
+    s.add_argument("object", choices=["request", "verdict"], nargs="?", default="verdict")
+    s.set_defaults(func=cmd_schema)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agentjury/judges/__init__.py

@@ -0,0 +1,14 @@
+from .base import ROLES, RUBRIC_VERSION, Completion, Judge, JudgeOpinion, load_roles, parse_opinion, register_roles
+from .fake import FakeJudge
+
+__all__ = ["ROLES", "RUBRIC_VERSION", "Completion", "Judge", "JudgeOpinion", "load_roles", "parse_opinion", "register_roles", "FakeJudge"]
+
+
+def openai_judge(role: str, model: str | None = None):
+    from .openai_judge import OpenAIJudge
+    return OpenAIJudge(role, model) if model else OpenAIJudge(role)
+
+
+def anthropic_judge(role: str, model: str | None = None):
+    from .anthropic_judge import AnthropicJudge
+    return AnthropicJudge(role, model) if model else AnthropicJudge(role)

agentjury/judges/anthropic_judge.py

@@ -0,0 +1,75 @@
+"""
+Judge backed by an Anthropic model.
+
+Requires ANTHROPIC_API_KEY in the environment.
+
+Workspace: if the key spans multiple workspaces, Anthropic requires an
+`anthropic-workspace-id` header; set ANTHROPIC_WORKSPACE_ID and it is sent.
+
+Thinking: Claude Sonnet 5 and later reason before answering by default, and
+that reasoning counts against max_tokens. We give the model 4096 tokens and
+default effort to "medium", which is plenty for a short JSON review.
+  ANTHROPIC_EFFORT     low | medium | high   (default medium)
+  ANTHROPIC_THINKING   adaptive | disabled   (default adaptive; Fable/Mythos reject disabled)
+"""
+
+from __future__ import annotations
+
+import os
+
+from .base import Completion, Judge
+
+
+class AnthropicJudge(Judge):
+    provider = "anthropic"
+
+    def __init__(self, role: str, model: str = "claude-sonnet-5", max_tokens: int = 4096,
+                 effort: str | None = None, thinking: str | None = None, timeout: float = 90.0):
+        super().__init__(role, model, timeout=timeout)
+        try:
+            from anthropic import Anthropic
+        except ImportError as exc:  # optional dependency
+            raise ImportError(
+                'The anthropic package is not installed. Run: pip install "agentjury[anthropic]"'
+            ) from exc
+
+        headers = {}
+        workspace_id = os.environ.get("ANTHROPIC_WORKSPACE_ID")
+        if workspace_id:
+            headers["anthropic-workspace-id"] = workspace_id
+
+        # max_retries=0: AgentJury owns the retry policy, not the SDK.
+        self._client = Anthropic(default_headers=headers, timeout=timeout, max_retries=0)
+        self.max_tokens = max_tokens
+        self.effort = effort or os.environ.get("ANTHROPIC_EFFORT", "medium")
+        self.thinking = thinking or os.environ.get("ANTHROPIC_THINKING", "adaptive")
+        self.params = {"max_tokens": max_tokens, "effort": self.effort, "thinking": self.thinking}
+
+    def complete(self, system: str, user: str) -> Completion:
+        kwargs = dict(
+            model=self.model,
+            max_tokens=self.max_tokens,
+            system=system,
+            messages=[{"role": "user", "content": user}],
+            output_config={"effort": self.effort},
+        )
+        if self.thinking == "disabled":
+            kwargs["thinking"] = {"type": "disabled"}
+
+        response = self._client.messages.create(**kwargs)
+        text = "".join(block.text for block in response.content if block.type == "text")
+
+        if not text.strip():
+            kinds = [block.type for block in response.content]
+            raise RuntimeError(
+                f"{self.model} returned no text (stop_reason={response.stop_reason}, blocks={kinds}). "
+                f"If stop_reason is max_tokens, raise max_tokens or lower ANTHROPIC_EFFORT."
+            )
+
+        usage = response.usage
+        return Completion(
+            text=text,
+            tokens_in=getattr(usage, "input_tokens", None),
+            tokens_out=getattr(usage, "output_tokens", None),
+            response_id=getattr(response, "id", None),
+        )