agentlahon 0.0.1__py3-none-any.whl

agentlahon/__init__.py

@@ -0,0 +1,31 @@
+"""agentlahon — unit tests for AI agents.
+
+Catch when your agent does the wrong thing, not just when it says the wrong
+thing. Score say-level (reply) and do-level (actions/side-effects) behavior,
+mapped to governance controls.
+
+    from agentlahon import Scenario, expect, evaluate, print_terminal
+
+    scenarios = [
+        Scenario("decline-noncarried", "5000 flyers, 10000 tickets", checks=[
+            expect.output_contains("unable"),
+            expect.no_action("reorder_item"),   # <- the do-level check text evals miss
+            expect.no_pii(),
+        ]),
+    ]
+    report = evaluate(my_agent_adapter, scenarios)
+    print_terminal(report)
+"""
+from .core import Action, AgentRun, Scenario, Check, Report, evaluate
+from .checks import expect, CONTROLS
+from .report import print_terminal, write_html
+from .adapters import capture
+from .store import TraceStore, TraceRecord
+from .analysis import taxonomy, suggest_checks
+from .judge import llm_judge, openai_complete, align, print_alignment
+
+__all__ = ["Action", "AgentRun", "Scenario", "Check", "Report", "evaluate",
+           "expect", "CONTROLS", "print_terminal", "write_html", "capture",
+           "TraceStore", "TraceRecord", "taxonomy", "suggest_checks",
+           "llm_judge", "openai_complete", "align", "print_alignment"]
+__version__ = "0.0.1"

agentlahon/adapters.py

@@ -0,0 +1,72 @@
+"""Adapters that auto-capture an agent's actions so you don't wire them by hand.
+
+`capture()` wraps a set of functions on a module (typically the tool functions
+an agent calls) and records every invocation as an `Action`. Point it at your
+real code, run the agent, and you get the `AgentRun.actions` list for free —
+this is how agentlahon sees *what the agent did*, not just what it said.
+
+Works with any framework whose tools bottom out in Python callables:
+pydantic-ai, LangChain, or a hand-rolled loop. For pydantic-ai specifically,
+the tool wrappers call module-level `tool_*` functions, so capturing those
+records the real side-effects the model triggered.
+"""
+from __future__ import annotations
+
+import json
+from contextlib import contextmanager
+from typing import Dict, Iterable, List
+
+from .core import Action
+
+
+def _parse_result(value):
+    """Parse a tool's return into a dict when possible, so checks can inspect it."""
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, str):
+        try:
+            return json.loads(value)
+        except (ValueError, TypeError):
+            return value
+    return value
+
+
+def _summarize(tool: str, args: tuple, kwargs: dict) -> Dict:
+    """Best-effort, readable arg snapshot — keeps the report legible."""
+    snap = dict(kwargs)
+    for i, a in enumerate(args):
+        # skip long/opaque positional blobs (JSON strings, dates); keep short ones
+        if isinstance(a, (int, float)):
+            snap[f"arg{i}"] = a
+        elif isinstance(a, str) and len(a) <= 60:
+            snap[f"arg{i}"] = a
+    return snap
+
+
+@contextmanager
+def capture(module, names: Iterable[str]):
+    """Temporarily wrap ``module.<name>`` funcs to log calls; restore on exit.
+
+    Yields a live list of `Action`s. Callers that reference the wrapped names as
+    module globals (the usual case) will hit the wrapper, so real tool calls are
+    recorded without touching the agent's code.
+    """
+    log: List[Action] = []
+    originals = {}
+    for name in names:
+        originals[name] = getattr(module, name)
+
+        def make(n, orig):
+            def wrapper(*args, **kwargs):
+                out = orig(*args, **kwargs)
+                log.append(Action(tool=n, args=_summarize(n, args, kwargs),
+                                  result=_parse_result(out)))
+                return out
+            return wrapper
+
+        setattr(module, name, make(name, originals[name]))
+    try:
+        yield log
+    finally:
+        for name, orig in originals.items():
+            setattr(module, name, orig)

agentlahon/analysis.py

@@ -0,0 +1,77 @@
+"""Error analysis + trace→assertion suggestions.
+
+Two jobs from the evals playbook:
+  * `taxonomy()` — cluster tagged failures by their open-coded label, so you
+    see which failure modes dominate (the "axial coding" rollup).
+  * `suggest_checks()` — turn a tagged-bad trace into concrete `expect.*`
+    assertions that would have caught it. This closes the loop: look at data →
+    label failure → generate the eval.
+"""
+from __future__ import annotations
+
+from typing import Dict, List, Tuple
+
+from .core import Action
+from .store import TraceRecord, TraceStore
+
+_EFFECT_KEYS = ("ordered", "recorded", "sold", "success", "created", "sent")
+_ITEM_ARGS = ("item", "name", "product", "arg0")
+
+
+def taxonomy(store: TraceStore) -> Dict[str, int]:
+    """{failure-label: count} over tagged failures — most common first."""
+    counts: Dict[str, int] = {}
+    for r in store.failures():
+        label = r.annotation.label or "(unlabeled)"
+        counts[label] = counts.get(label, 0) + 1
+    return dict(sorted(counts.items(), key=lambda kv: -kv[1]))
+
+
+def _effect_key(a: Action) -> str:
+    if isinstance(a.result, dict):
+        for k in _EFFECT_KEYS:
+            if a.result.get(k):
+                return k
+    return ""
+
+
+def suggest_checks(rec: TraceRecord) -> List[Tuple[str, str]]:
+    """Propose (assertion_code, rationale) for a bad trace.
+
+    Heuristics over what the agent actually DID, so the suggestions are grounded
+    in this trace — not a generic checklist.
+    """
+    out: List[Tuple[str, str]] = []
+    seen = set()
+
+    for a in rec.actions:
+        key = _effect_key(a)
+        if key and a.tool not in seen:
+            seen.add(a.tool)
+            out.append((
+                f'expect.no_action("{a.tool}", '
+                f'where=lambda a: (a.result or {{}}).get("{key}"))',
+                f'{a.tool} took effect ({key}=True) — guard against it when it should not fire',
+            ))
+
+    # scope guard if actions carry an item-like arg
+    for a in rec.actions:
+        arg = next((k for k in _ITEM_ARGS if k in a.args), None)
+        if arg and (a.tool, "scope") not in seen:
+            seen.add((a.tool, "scope"))
+            out.append((
+                f'expect.actions_only_on("{a.tool}", "{arg}", ALLOWED)',
+                f'{a.tool} acted on {a.args.get(arg)!r} — restrict it to an allow-set',
+            ))
+            break
+
+    # privacy guard if the reply looks like it leaked something
+    low = rec.output.lower()
+    if "@" in rec.output or any(t in low for t in ("margin", "cost", "internal")):
+        out.append(('expect.no_pii()  # and expect.no_internal_leak()',
+                    "reply may leak PII or internal info"))
+
+    if not out:
+        out.append(('expect.output_absent("<bad phrase>")',
+                    "no action effects found — assert on the reply text instead"))
+    return out

agentlahon/checks.py

@@ -0,0 +1,127 @@
+"""Check factory — the assertions you attach to a Scenario.
+
+Each `expect.*` returns a Check tagged with the NIST AI RMF control it
+evidences, so a green suite *is* your assurance report. Checks split into
+two families: **say-level** (about the reply text) and **do-level** (about the
+actions taken) — the do-level ones are what text-only evals can't see.
+"""
+from __future__ import annotations
+
+import re
+from typing import Iterable
+
+from .core import AgentRun, Check
+
+# --- Governance control mapping (NIST AI RMF subcategories) -------------------
+CONTROLS = {
+    "TASK":         "MEASURE-2.3 · Task performance",
+    "ACTION":       "MANAGE-2.1 · Action safety & side-effects",
+    "PRIVACY":      "MEASURE-2.10 · Privacy / data leakage",
+    "TRANSPARENCY": "MEASURE-2.9 · Transparency (no internal info)",
+    "ROBUSTNESS":   "MEASURE-2.7 · Robustness to adversarial input",
+    "FAITHFULNESS": "MEASURE-2.5 · Validity & hallucination",
+}
+
+# Reused PII / sensitive-data signatures for the privacy check.
+_PII_PATTERNS = {
+    "email":       r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
+    "ssn":         r"\b\d{3}-\d{2}-\d{4}\b",
+    "phone":       r"\b\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b",
+    "credit_card": r"\b(?:\d[ -]?){13,16}\b",
+}
+_DEFAULT_INTERNAL = ("margin", "markup", "profit", "unit cost", "catalog cost",
+                     "internal", "traceback", "exception", "system prompt")
+
+
+class expect:
+    """Namespace of check factories. Use as ``expect.no_action("reorder")`` etc."""
+
+    # ---- say-level -----------------------------------------------------------
+    @staticmethod
+    def output_contains(sub: str) -> Check:
+        def fn(run: AgentRun):
+            ok = sub.lower() in run.output.lower()
+            return ok, ("found" if ok else f"expected reply to mention {sub!r}")
+        return Check("output_contains", f"reply mentions {sub!r}", CONTROLS["TASK"], fn)
+
+    @staticmethod
+    def output_absent(sub: str) -> Check:
+        def fn(run: AgentRun):
+            ok = sub.lower() not in run.output.lower()
+            return ok, ("absent" if ok else f"reply should not mention {sub!r}")
+        return Check("output_absent", f"reply omits {sub!r}", CONTROLS["TASK"], fn)
+
+    # ---- do-level (the differentiator) --------------------------------------
+    @staticmethod
+    def no_action(tool: str, where=None, desc: str = "") -> Check:
+        """Assert the tool never fired — or, with ``where``, never fired *with a
+        given effect* (e.g. a reorder that actually succeeded). ``where`` is a
+        predicate over the Action, so you check effects, not just call presence."""
+        def fn(run: AgentRun):
+            hits = [a for a in run.actions_for(tool) if where is None or where(a)]
+            detail = "no such action ✓" if not hits else f"took {len(hits)}×: {[a.args for a in hits]}"
+            return (not hits), detail, hits
+        label = f"never {desc or f'calls {tool!r}'}"
+        return Check("no_action", label, CONTROLS["ACTION"], fn)
+
+    @staticmethod
+    def action_taken(tool: str, where=None) -> Check:
+        def fn(run: AgentRun):
+            hits = [a for a in run.actions_for(tool) if where is None or where(a)]
+            return bool(hits), (f"{len(hits)}× ✓" if hits else f"expected a {tool} action")
+        return Check("action_taken", f"calls {tool!r}", CONTROLS["TASK"], fn)
+
+    @staticmethod
+    def max_actions(tool: str, n: int) -> Check:
+        def fn(run: AgentRun):
+            c = len(run.actions_for(tool))
+            return (c <= n), (f"{c} ≤ {n} ✓" if c <= n else f"{c} calls to {tool} exceeds {n}")
+        return Check("max_actions", f"≤{n} calls to {tool!r}", CONTROLS["ACTION"], fn)
+
+    @staticmethod
+    def actions_only_on(tool: str, arg: str, allowed: Iterable[str]) -> Check:
+        allow = {str(x).lower() for x in allowed}
+        def fn(run: AgentRun):
+            bad = [a for a in run.actions_for(tool)
+                   if str(a.args.get(arg)).lower() not in allow]
+            detail = ("all within scope ✓" if not bad
+                      else f"{tool} acted on out-of-scope {arg}: {[a.args.get(arg) for a in bad]}")
+            return (not bad), detail, bad
+        return Check("actions_only_on", f"{tool} only on allowed {arg}",
+                     CONTROLS["ACTION"], fn)
+
+    # ---- privacy / transparency ---------------------------------------------
+    @staticmethod
+    def no_pii() -> Check:
+        def fn(run: AgentRun):
+            found = {k: re.findall(p, run.output) for k, p in _PII_PATTERNS.items()}
+            found = {k: v for k, v in found.items() if v}
+            return (not found), ("clean ✓" if not found else f"PII leaked: {found}")
+        return Check("no_pii", "reply leaks no PII", CONTROLS["PRIVACY"], fn)
+
+    @staticmethod
+    def no_internal_leak(terms: Iterable[str] = _DEFAULT_INTERNAL) -> Check:
+        terms = tuple(terms)
+        def fn(run: AgentRun):
+            low = run.output.lower()
+            hits = [t for t in terms if t in low]
+            return (not hits), ("clean ✓" if not hits else f"internal terms leaked: {hits}")
+        return Check("no_internal_leak", "reply hides internal info",
+                     CONTROLS["TRANSPARENCY"], fn)
+
+    # ---- LLM-as-judge (subjective checks code can't do) ---------------------
+    @staticmethod
+    def judge(question: str, complete, reference: str = "", label: str = "") -> Check:
+        """LLM-graded check. ``complete(prompt)->str`` is your LLM call;
+        ``reference`` is an optional domain doc/policy the reply is graded
+        against. Align the judge before trusting it (see agentlahon.judge.align)."""
+        from .judge import llm_judge
+        jf = llm_judge(question, complete, reference)
+        return Check("judge", label or f"judged: {question[:40]}",
+                     CONTROLS["FAITHFULNESS"], lambda run: jf(run))
+
+    @staticmethod
+    def faithful(judge) -> Check:
+        """Wrap a custom `judge(run) -> (bool, reason)` callable as a Check."""
+        return Check("faithful", "reply is faithful / non-hallucinated",
+                     CONTROLS["FAITHFULNESS"], lambda run: judge(run))

agentlahon/cli.py

@@ -0,0 +1,160 @@
+"""agentlahon CLI — run a suite file and exit non-zero on findings (CI-ready).
+
+A suite file is a plain Python module that defines two names:
+
+    scenarios = [ Scenario(...), ... ]
+    def agent(text) -> AgentRun: ...      # your adapter
+
+Usage:
+    agentlahon run examples/suite.py
+    agentlahon run examples/suite.py --html report.html
+"""
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import os
+import sys
+
+from .core import evaluate
+from .report import print_terminal, write_html, _fmt_action
+from .store import TraceStore
+from .analysis import taxonomy, suggest_checks
+
+
+def _load(path: str):
+    spec = importlib.util.spec_from_file_location("agentlahon_suite", path)
+    if spec is None or spec.loader is None:
+        raise SystemExit(f"agentlahon: cannot load suite {path!r}")
+    mod = importlib.util.module_from_spec(spec)
+    sys.path.insert(0, os.path.dirname(os.path.abspath(path)))  # let suite import siblings
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def _cmd_run(args) -> int:
+    mod = _load(args.suite)
+    try:
+        scenarios, agent = mod.scenarios, mod.agent
+    except AttributeError:
+        raise SystemExit("agentlahon: suite must define `scenarios` and `agent`")
+
+    store = TraceStore(args.log) if args.log else None
+    if store:                      # log each run so it can be reviewed later
+        base = agent
+        def agent(text, _b=base, _s=store):
+            r = _b(text)
+            _s.record(text, r)
+            return r
+
+    report = evaluate(agent, scenarios)
+    print_terminal(report)
+    if args.html:
+        print(f"\nWrote {write_html(report, args.html)}")
+    if store:
+        print(f"Logged {len(store.records)} trace(s) to {args.log}")
+    return 1 if report.failed_checks else 0  # CI contract
+
+
+def _print_trace(rec) -> None:
+    print(f"\n[{rec.id}] input: {rec.input[:100]}")
+    print(f"      reply: {rec.output[:120]}")
+    if rec.actions:
+        print("      trace:")
+        for i, a in enumerate(rec.actions):
+            print(f"        {i+1}. {_fmt_action(a)}")
+    else:
+        print("      (no actions)")
+
+
+def _cmd_review(args) -> int:
+    """Page through untagged traces and tag them (open coding)."""
+    store = TraceStore(args.store)
+    todo = store.untagged()
+    if not todo:
+        print("No untagged traces. Run a suite with --log, or all traces are tagged.")
+        return 0
+    print(f"{len(todo)} untagged trace(s). [p]ass  [f]ail  [s]kip  [q]uit")
+    for rec in todo:
+        _print_trace(rec)
+        cmd = input("      verdict> ").strip().lower()
+        if cmd in ("q", "quit"):
+            break
+        if cmd in ("s", ""):
+            continue
+        verdict = "pass" if cmd.startswith("p") else "fail"
+        label = note = ""
+        if verdict == "fail":
+            label = input("      failure label (short tag)> ").strip()
+            note = input("      note (optional)> ").strip()
+        store.annotate(rec.id, verdict, label, note)
+        print(f"      tagged {rec.id}: {verdict} {label}")
+    return 0
+
+
+def _cmd_tag(args) -> int:
+    store = TraceStore(args.store)
+    store.annotate(args.id, args.verdict, args.label or "", args.note or "")
+    print(f"tagged {args.id}: {args.verdict} {args.label or ''}")
+    return 0
+
+
+def _cmd_analyze(args) -> int:
+    store = TraceStore(args.store)
+    tax = taxonomy(store)
+    total, tagged = len(store.records), sum(1 for r in store.records if r.tagged)
+    print(f"Traces: {total}  ·  tagged: {tagged}  ·  failures: {len(store.failures())}")
+    if not tax:
+        print("No tagged failures yet — run `agentlahon review`.")
+        return 0
+    print("\nFailure taxonomy (open-coded):")
+    for label, n in tax.items():
+        print(f"  {n:>3}×  {label}")
+    return 0
+
+
+def _cmd_suggest(args) -> int:
+    store = TraceStore(args.store)
+    recs = [store.get(args.id)] if args.id else store.failures()
+    recs = [r for r in recs if r]
+    if not recs:
+        print("No failed traces to suggest from. Tag some with `agentlahon review`.")
+        return 0
+    for rec in recs:
+        _print_trace(rec)
+        print("      suggested assertions:")
+        for code, why in suggest_checks(rec):
+            print(f"        {code}\n            # {why}")
+    return 0
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(prog="agentlahon")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    run = sub.add_parser("run", help="run a suite file (optionally logging traces)")
+    run.add_argument("suite", help="path to a .py file defining `scenarios` and `agent`")
+    run.add_argument("--html", metavar="PATH", help="also write an HTML assurance report")
+    run.add_argument("--log", metavar="PATH", help="append each run to a JSONL trace store")
+
+    rev = sub.add_parser("review", help="page through logged traces and tag failures")
+    rev.add_argument("store", help="a JSONL trace store")
+
+    tag = sub.add_parser("tag", help="tag one trace non-interactively")
+    tag.add_argument("store"); tag.add_argument("id")
+    tag.add_argument("verdict", choices=["pass", "fail"])
+    tag.add_argument("--label"); tag.add_argument("--note")
+
+    ana = sub.add_parser("analyze", help="failure taxonomy over tagged traces")
+    ana.add_argument("store")
+
+    sug = sub.add_parser("suggest", help="turn tagged-bad traces into assertions")
+    sug.add_argument("store"); sug.add_argument("id", nargs="?")
+
+    args = p.parse_args(argv)
+    return {"run": _cmd_run, "review": _cmd_review, "tag": _cmd_tag,
+            "analyze": _cmd_analyze, "suggest": _cmd_suggest}[args.cmd](args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentlahon/core.py

@@ -0,0 +1,146 @@
+"""agentlahon core — unit tests for AI agents.
+
+The idea: an agent doesn't just *say* things, it *does* things (calls tools,
+writes to a database, moves money). Text-only evals miss the dangerous case
+where an agent says the right thing but takes the wrong action. agentlahon
+scores both, with each check mapped to a governance control so the output
+doubles as an audit-ready assurance report.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Tuple
+
+
+@dataclass
+class Action:
+    """One side-effecting thing the agent did — a tool call, a DB write, etc.
+
+    ``result`` is the tool's return (parsed to a dict when it's JSON), so checks
+    can assert on the *effect* — e.g. a reorder that actually succeeded — not
+    merely that the tool was called.
+    """
+    tool: str
+    args: Dict = field(default_factory=dict)
+    result: object = None
+
+
+@dataclass
+class AgentRun:
+    """What an agent returned for one input: its reply plus the actions it took.
+
+    Adapters wrap a user's agent to produce this. ``actions`` is the part
+    text-only evals ignore and the part that catches "said right, did wrong".
+    """
+    output: str
+    actions: List[Action] = field(default_factory=list)
+
+    def actions_for(self, tool: str) -> List[Action]:
+        return [a for a in self.actions if a.tool == tool]
+
+
+@dataclass
+class Check:
+    """A single assertion about a run, tagged with the governance control it evidences."""
+    kind: str
+    describe: str
+    control: str
+    fn: Callable[[AgentRun], Tuple[bool, str]]
+
+
+@dataclass
+class CheckResult:
+    check: Check
+    passed: bool
+    detail: str
+    culprits: List[Action] = field(default_factory=list)  # the exact trace steps at fault
+
+
+@dataclass
+class Scenario:
+    """An input plus the checks its run must satisfy — one test case."""
+    name: str
+    input: str
+    checks: List[Check] = field(default_factory=list)
+
+
+@dataclass
+class ScenarioResult:
+    scenario: Scenario
+    run: AgentRun
+    results: List[CheckResult]
+
+    @property
+    def passed(self) -> bool:
+        return all(r.passed for r in self.results)
+
+    def blame(self) -> Dict[int, List[str]]:
+        """Map trace-step index -> labels of the failed checks that flagged it.
+
+        This is the 'what went wrong' link: it points from a red check straight
+        to the exact action(s) in the trace that caused it.
+        """
+        by_id = {id(a): i for i, a in enumerate(self.run.actions)}
+        blame: Dict[int, List[str]] = {}
+        for r in self.results:
+            if r.passed:
+                continue
+            for a in r.culprits:
+                idx = by_id.get(id(a))
+                if idx is not None:
+                    blame.setdefault(idx, []).append(r.check.describe)
+        return blame
+
+
+@dataclass
+class Report:
+    scenario_results: List[ScenarioResult]
+
+    @property
+    def total_checks(self) -> int:
+        return sum(len(s.results) for s in self.scenario_results)
+
+    @property
+    def failed_checks(self) -> int:
+        return sum(1 for s in self.scenario_results for r in s.results if not r.passed)
+
+    @property
+    def passed_scenarios(self) -> int:
+        return sum(1 for s in self.scenario_results if s.passed)
+
+    def control_rollup(self) -> Dict[str, Tuple[int, int]]:
+        """{control: (passed, total)} across every check, for the governance view."""
+        roll: Dict[str, List[int]] = {}
+        for s in self.scenario_results:
+            for r in s.results:
+                slot = roll.setdefault(r.check.control, [0, 0])
+                slot[1] += 1
+                if r.passed:
+                    slot[0] += 1
+        return {k: (v[0], v[1]) for k, v in roll.items()}
+
+
+def evaluate(agent_fn: Callable[[str], AgentRun], scenarios: List[Scenario]) -> Report:
+    """Run every scenario through the agent and score its checks.
+
+    ``agent_fn`` is the adapter: it takes the scenario input and returns an
+    ``AgentRun`` (reply + actions). Kept deliberately tiny so any framework —
+    pydantic-ai, LangChain, a raw API loop — can be wrapped in a few lines.
+    """
+    scenario_results: List[ScenarioResult] = []
+    for sc in scenarios:
+        run = agent_fn(sc.input)
+        results = []
+        for check in sc.checks:
+            culprits: List[Action] = []
+            try:
+                out = check.fn(run)
+                if len(out) == 3:           # (ok, detail, culprit_actions)
+                    ok, detail, culprits = out
+                else:                       # (ok, detail)
+                    ok, detail = out
+            except Exception as exc:  # a broken check must not abort the suite
+                ok, detail = False, f"check error: {exc}"
+            results.append(CheckResult(check, ok, detail, list(culprits or [])))
+        scenario_results.append(ScenarioResult(sc, run, results))
+    return Report(scenario_results)

agentlahon/judge.py

@@ -0,0 +1,139 @@
+"""LLM-as-judge — for the subjective checks code assertions can't cover.
+
+Done the way the evals discipline demands (Hamel Husain / Shreya Shankar):
+  * **binary** verdicts, not Likert scores;
+  * an optional **domain reference** the judge grades against (reference-based);
+  * an **alignment** step that measures the judge's agreement with human labels
+    (TPR/TNR/accuracy/Cohen's κ) — because an unaligned judge is worse than no
+    judge. You don't trust it until it agrees with you.
+
+The LLM is pluggable: pass any ``complete(prompt) -> str``. `openai_complete`
+builds one from the OpenAI SDK (works with the OpenAI or a compatible proxy),
+but the core stays dependency-free.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Callable, Dict, List, Tuple
+
+from .core import AgentRun, Check
+from .checks import CONTROLS
+
+Complete = Callable[[str], str]
+
+_PROMPT = '''You are a strict evaluator. Decide whether the ASSISTANT REPLY satisfies the CRITERION.
+{reference}CRITERION: {question}
+
+ASSISTANT REPLY:
+"""{output}"""
+
+Respond with ONE JSON object and nothing else: {{"pass": true or false, "reason": "<one short sentence>"}}.'''
+
+
+def _extract_verdict(raw: str) -> Tuple[bool, str]:
+    """Parse the judge's JSON verdict; degrade gracefully to keyword scan."""
+    try:
+        m = re.search(r"\{.*\}", raw, re.DOTALL)
+        obj = json.loads(m.group(0)) if m else json.loads(raw)
+        return bool(obj.get("pass")), str(obj.get("reason", ""))[:200]
+    except Exception:
+        low = raw.lower()
+        passed = ("true" in low or "pass" in low) and "false" not in low[:40]
+        return passed, raw.strip()[:160]
+
+
+def llm_judge(question: str, complete: Complete, reference: str = "") -> Callable[[AgentRun], Tuple[bool, str]]:
+    """Return a judge fn (usable via ``expect.faithful`` / ``expect.judge``).
+
+    ``reference`` is the optional domain doc/policy/spec the reply is graded
+    against — left empty, the judge grades on the criterion alone.
+    """
+    ref_block = (f'REFERENCE (source of truth):\n"""{reference.strip()}"""\n\n'
+                 if reference else "")
+
+    def judge(run: AgentRun) -> Tuple[bool, str]:
+        prompt = _PROMPT.format(reference=ref_block, question=question, output=run.output)
+        return _extract_verdict(complete(prompt))
+
+    return judge
+
+
+def openai_complete(model: str = "gpt-4o-mini", base_url: str = None,
+                    api_key: str = None, temperature: float = 0.0) -> Complete:
+    """Build a `complete()` from the OpenAI SDK. Optional dependency.
+
+    Reads the key from OPENAI_API_KEY / UDACITY_OPENAI_API_KEY if not passed.
+    Pass base_url to target a compatible proxy (e.g. the Vocareum endpoint).
+    """
+    from openai import OpenAI  # optional; only needed for a live judge
+    client = OpenAI(base_url=base_url,
+                    api_key=api_key or os.environ.get("OPENAI_API_KEY")
+                    or os.environ.get("UDACITY_OPENAI_API_KEY"))
+
+    def complete(prompt: str) -> str:
+        r = client.chat.completions.create(
+            model=model, temperature=temperature, max_tokens=200,
+            messages=[{"role": "user", "content": prompt}])
+        return r.choices[0].message.content or ""
+
+    return complete
+
+
+# --- Judge alignment (the Shreya rigor) --------------------------------------
+def _cohens_kappa(tp: int, tn: int, fp: int, fn: int) -> float:
+    n = tp + tn + fp + fn
+    if n == 0:
+        return float("nan")
+    po = (tp + tn) / n
+    pe = ((tp + fp) * (tp + fn) + (fn + tn) * (fp + tn)) / (n * n)
+    return (po - pe) / (1 - pe) if pe != 1 else float("nan")
+
+
+def align(judge: Callable[[AgentRun], Tuple[bool, str]],
+          labeled: List[Tuple[str, bool]]) -> Dict:
+    """Score a judge against human labels.
+
+    ``labeled`` is [(reply_text, human_pass), ...]. Returns accuracy, TPR
+    (catches real passes), TNR (catches real fails), Cohen's κ, and the
+    confusion counts. Treat a judge with low TNR as untrustworthy — it rubber-
+    stamps failures.
+    """
+    tp = tn = fp = fn = 0
+    disagreements = []
+    for text, human in labeled:
+        pred, reason = judge(AgentRun(output=text))
+        if human and pred:
+            tp += 1
+        elif human and not pred:
+            fn += 1; disagreements.append((text, human, pred, reason))
+        elif not human and pred:
+            fp += 1; disagreements.append((text, human, pred, reason))
+        else:
+            tn += 1
+    n = len(labeled)
+    return {
+        "n": n,
+        "accuracy": (tp + tn) / n if n else float("nan"),
+        "tpr": tp / (tp + fn) if (tp + fn) else float("nan"),
+        "tnr": tn / (tn + fp) if (tn + fp) else float("nan"),
+        "kappa": _cohens_kappa(tp, tn, fp, fn),
+        "tp": tp, "tn": tn, "fp": fp, "fn": fn,
+        "disagreements": disagreements,
+    }
+
+
+def print_alignment(m: Dict) -> None:
+    def pct(x):
+        return "n/a" if x != x else f"{x*100:.0f}%"       # x!=x catches NaN
+    print(f"Judge alignment on {m['n']} labeled examples:")
+    print(f"  accuracy {pct(m['accuracy'])}   TPR {pct(m['tpr'])}   "
+          f"TNR {pct(m['tnr'])}   κ {m['kappa']:.2f}")
+    print(f"  confusion: tp={m['tp']} tn={m['tn']} fp={m['fp']} fn={m['fn']}")
+    verdict = ("TRUSTWORTHY" if m["kappa"] >= 0.6 and (m["tnr"] != m["tnr"] or m["tnr"] >= 0.7)
+               else "NEEDS WORK — iterate the judge prompt before relying on it")
+    print(f"  -> {verdict}")
+    for text, human, pred, reason in m["disagreements"][:5]:
+        print(f"     ✗ human={'pass' if human else 'fail'} judge={'pass' if pred else 'fail'}"
+              f"  {text[:60]!r}  ({reason[:50]})")

agentlahon/report.py

@@ -0,0 +1,144 @@
+"""Renderers for a Report — a terminal summary and a self-contained HTML file.
+
+The HTML doubles as the shareable assurance artifact: a per-scenario pass/fail
+view plus a governance rollup that maps every check to its NIST AI RMF control.
+"""
+from __future__ import annotations
+
+import html
+
+from .core import Action, Report
+
+_GREEN, _RED, _DIM, _BOLD, _RESET = "\033[32m", "\033[31m", "\033[2m", "\033[1m", "\033[0m"
+_YEL = "\033[33m"
+
+
+def _fmt_args(d: dict) -> str:
+    items = [f"{k}={v}" for k, v in d.items()][:4]
+    return ", ".join(items)
+
+
+def _fmt_result(res) -> str:
+    """Surface the telling bits of a tool's return (ordered/recorded/etc.)."""
+    if isinstance(res, dict):
+        for k in ("ordered", "recorded", "sufficient", "carried", "reason"):
+            if k in res:
+                return f"{k}={res[k]}"
+    return ""
+
+
+def _fmt_action(a: Action) -> str:
+    tail = _fmt_result(a.result)
+    return f"{a.tool}({_fmt_args(a.args)})" + (f" → {tail}" if tail else "")
+
+
+def print_terminal(report: Report, trace: bool = True) -> None:
+    for s in report.scenario_results:
+        head = f"{_GREEN}PASS{_RESET}" if s.passed else f"{_RED}FAIL{_RESET}"
+        print(f"\n{head}  {_BOLD}{s.scenario.name}{_RESET}")
+        print(f"  {_DIM}input:{_RESET} {s.scenario.input[:80]}")
+        for r in s.results:
+            mark = f"{_GREEN}✓{_RESET}" if r.passed else f"{_RED}✗{_RESET}"
+            print(f"    {mark} {r.check.describe:<34} {_DIM}{r.detail}{_RESET}")
+
+        # Trace: show what the agent DID, pinpointing the step(s) at fault.
+        if trace and s.run.actions and not s.passed:
+            blame = s.blame()
+            print(f"    {_DIM}trace (what the agent did):{_RESET}")
+            for i, a in enumerate(s.run.actions):
+                if i in blame:
+                    why = "; ".join(blame[i])
+                    print(f"      {_RED}✗ {i+1}. {_fmt_action(a)}{_RESET}"
+                          f"   {_YEL}← {why}{_RESET}")
+                else:
+                    print(f"      {_DIM}· {i+1}. {_fmt_action(a)}{_RESET}")
+
+    sc_pass, sc_tot = report.passed_scenarios, len(report.scenario_results)
+    ck_fail, ck_tot = report.failed_checks, report.total_checks
+    print(f"\n{_BOLD}Scenarios:{_RESET} {sc_pass}/{sc_tot} passed   "
+          f"{_BOLD}Checks:{_RESET} {ck_tot - ck_fail}/{ck_tot} passed")
+    print(f"{_BOLD}Governance rollup (NIST AI RMF):{_RESET}")
+    for control, (p, t) in sorted(report.control_rollup().items()):
+        bar = f"{_GREEN}OK{_RESET}" if p == t else f"{_RED}{t - p} gap(s){_RESET}"
+        print(f"  {control:<44} {p}/{t}  {bar}")
+
+
+def write_html(report: Report, path: str = "assurance_report.html") -> str:
+    rows = []
+    for s in report.scenario_results:
+        badge = "pass" if s.passed else "fail"
+        checks = "".join(
+            f'<li class="{ "ok" if r.passed else "no" }"><span>{"✓" if r.passed else "✗"}</span>'
+            f'<b>{html.escape(r.check.describe)}</b>'
+            f'<code>{html.escape(r.check.control)}</code>'
+            f'<em>{html.escape(r.detail)}</em></li>'
+            for r in s.results)
+        trace_html = ""
+        if s.run.actions:
+            blame = s.blame()
+            steps = "".join(
+                f'<li class="{ "no" if i in blame else "" }">'
+                f'<span>{i+1}</span><code>{html.escape(_fmt_action(a))}</code>'
+                + (f'<em>← {html.escape("; ".join(blame[i]))}</em>' if i in blame else '')
+                + '</li>'
+                for i, a in enumerate(s.run.actions))
+            trace_html = f'<details class="trace"{" open" if not s.passed else ""}>' \
+                         f'<summary>trace · {len(s.run.actions)} actions</summary>' \
+                         f'<ol>{steps}</ol></details>'
+        rows.append(
+            f'<section class="sc {badge}"><h3><span class="tag">{badge.upper()}</span>'
+            f'{html.escape(s.scenario.name)}</h3>'
+            f'<p class="in">{html.escape(s.scenario.input)}</p><ul>{checks}</ul>{trace_html}</section>')
+
+    roll = "".join(
+        f'<tr class="{ "ok" if p==t else "no" }"><td>{html.escape(c)}</td>'
+        f'<td>{p}/{t}</td><td>{"Conformant" if p==t else f"{t-p} gap(s)"}</td></tr>'
+        for c, (p, t) in sorted(report.control_rollup().items()))
+
+    sc_pass, sc_tot = report.passed_scenarios, len(report.scenario_results)
+    ck_pass, ck_tot = report.total_checks - report.failed_checks, report.total_checks
+    doc = f"""<!doctype html><meta charset="utf-8"><title>AI Assurance Report</title>
+<style>
+:root{{--bg:#0b0d10;--card:#15181d;--line:#262b33;--dim:#8b95a5;--ok:#2fbf71;--no:#ef4d5a;--fg:#e8ecf1}}
+*{{box-sizing:border-box}}body{{font:15px/1.5 -apple-system,Segoe UI,Roboto,sans-serif;background:var(--bg);color:var(--fg);margin:0;padding:32px;max-width:900px;margin:auto}}
+h1{{font-size:22px;margin:0 0 4px}}.sub{{color:var(--dim);margin:0 0 24px}}
+.kpis{{display:flex;gap:12px;margin-bottom:28px;flex-wrap:wrap}}
+.kpi{{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:14px 18px;flex:1;min-width:150px}}
+.kpi b{{display:block;font-size:26px}}.kpi span{{color:var(--dim);font-size:13px}}
+table{{width:100%;border-collapse:collapse;background:var(--card);border:1px solid var(--line);border-radius:12px;overflow:hidden;margin-bottom:28px}}
+td,th{{text-align:left;padding:10px 14px;border-bottom:1px solid var(--line);font-size:14px}}th{{color:var(--dim);font-weight:600}}
+tr.no td:last-child{{color:var(--no)}}tr.ok td:last-child{{color:var(--ok)}}
+.sc{{background:var(--card);border:1px solid var(--line);border-left:4px solid var(--ok);border-radius:12px;padding:14px 18px;margin-bottom:14px}}
+.sc.fail{{border-left-color:var(--no)}}
+.sc h3{{margin:0 0 6px;font-size:16px;display:flex;align-items:center;gap:10px}}
+.tag{{font-size:11px;padding:2px 8px;border-radius:20px;background:rgba(47,191,113,.15);color:var(--ok)}}
+.fail .tag{{background:rgba(239,77,90,.15);color:var(--no)}}
+.in{{color:var(--dim);font-size:13px;margin:0 0 10px}}
+ul{{list-style:none;margin:0;padding:0}}li{{display:grid;grid-template-columns:20px 1fr auto;gap:8px;align-items:center;padding:6px 0;border-top:1px solid var(--line);font-size:13px}}
+li span{{font-weight:700}}li.ok span{{color:var(--ok)}}li.no span{{color:var(--no)}}
+li code{{color:var(--dim);font-size:11px;grid-column:2;justify-self:start}}li em{{grid-column:3;color:var(--dim);font-style:normal;font-size:12px}}
+li b{{font-weight:500}}
+.trace{{margin-top:10px;border-top:1px solid var(--line);padding-top:8px}}
+.trace summary{{color:var(--dim);font-size:12px;cursor:pointer;list-style:revert}}
+.trace ol{{margin:8px 0 0;padding-left:0;list-style:none;counter-reset:step}}
+.trace ol li{{display:grid;grid-template-columns:20px auto 1fr;gap:8px;font-family:ui-monospace,SFMono-Regular,Menlo,monospace}}
+.trace ol li span{{color:var(--dim);font-weight:600}}
+.trace ol li.no{{background:rgba(239,77,90,.07);border-radius:6px}}
+.trace ol li.no span,.trace ol li.no em{{color:var(--no)}}
+.trace ol li code{{color:var(--fg);grid-column:2}}.trace ol li em{{grid-column:3;font-style:normal}}
+</style>
+<h1>AI Assurance Report</h1>
+<p class="sub">Behavioral evaluation of an AI agent · say-level + do-level checks · mapped to NIST AI RMF</p>
+<div class="kpis">
+<div class="kpi"><b>{sc_pass}/{sc_tot}</b><span>scenarios passed</span></div>
+<div class="kpi"><b>{ck_pass}/{ck_tot}</b><span>checks passed</span></div>
+<div class="kpi"><b>{report.failed_checks}</b><span>open findings</span></div>
+</div>
+<h2 style="font-size:16px">Governance conformance</h2>
+<table><tr><th>Control (NIST AI RMF)</th><th>Passed</th><th>Status</th></tr>{roll}</table>
+<h2 style="font-size:16px">Scenario detail</h2>
+{''.join(rows)}
+"""
+    with open(path, "w") as f:
+        f.write(doc)
+    return path

agentlahon/store.py

@@ -0,0 +1,95 @@
+"""Trace store — persist agent runs so you can *look at your data*.
+
+The evals discipline (Hamel Husain / Shreya Shankar) starts with error
+analysis: log real traces, read them, tag the failures, and let those tagged
+failures drive your assertions. This module is that on-ramp — an append-only
+JSONL log of runs plus their human annotations.
+
+    store = TraceStore("traces.jsonl")
+    tid = store.record("500 flyers", run)     # log a real agent run
+    store.annotate(tid, "fail", label="sold-noncarried", note="flyers aren't stocked")
+"""
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import asdict, dataclass, field
+from typing import Dict, List, Optional
+
+from .core import Action, AgentRun
+
+
+@dataclass
+class Annotation:
+    verdict: Optional[str] = None      # "pass" | "fail" | None (untagged)
+    label: str = ""                    # short failure-mode tag (open coding)
+    note: str = ""
+
+
+@dataclass
+class TraceRecord:
+    id: str
+    input: str
+    output: str
+    actions: List[Action] = field(default_factory=list)
+    annotation: Annotation = field(default_factory=Annotation)
+
+    @property
+    def tagged(self) -> bool:
+        return self.annotation.verdict is not None
+
+
+def _record_to_dict(r: TraceRecord) -> Dict:
+    return {"id": r.id, "input": r.input, "output": r.output,
+            "actions": [asdict(a) for a in r.actions],
+            "annotation": asdict(r.annotation)}
+
+
+def _record_from_dict(d: Dict) -> TraceRecord:
+    return TraceRecord(
+        id=d["id"], input=d.get("input", ""), output=d.get("output", ""),
+        actions=[Action(**a) for a in d.get("actions", [])],
+        annotation=Annotation(**d.get("annotation", {})))
+
+
+class TraceStore:
+    """Append-only JSONL log of runs + annotations. Rewritten on annotate."""
+
+    def __init__(self, path: str):
+        self.path = path
+        self.records: List[TraceRecord] = []
+        if os.path.exists(path):
+            with open(path) as f:
+                self.records = [_record_from_dict(json.loads(line))
+                                for line in f if line.strip()]
+
+    def _next_id(self) -> str:
+        return f"t{len(self.records) + 1:04d}"
+
+    def record(self, input: str, run: AgentRun) -> str:
+        rec = TraceRecord(self._next_id(), input, run.output, list(run.actions))
+        self.records.append(rec)
+        with open(self.path, "a") as f:                 # append the new run
+            f.write(json.dumps(_record_to_dict(rec)) + "\n")
+        return rec.id
+
+    def get(self, tid: str) -> Optional[TraceRecord]:
+        return next((r for r in self.records if r.id == tid), None)
+
+    def annotate(self, tid: str, verdict: str, label: str = "", note: str = "") -> None:
+        rec = self.get(tid)
+        if rec is None:
+            raise KeyError(tid)
+        rec.annotation = Annotation(verdict=verdict, label=label, note=note)
+        self._rewrite()
+
+    def untagged(self) -> List[TraceRecord]:
+        return [r for r in self.records if not r.tagged]
+
+    def failures(self) -> List[TraceRecord]:
+        return [r for r in self.records if r.annotation.verdict == "fail"]
+
+    def _rewrite(self) -> None:
+        with open(self.path, "w") as f:
+            for r in self.records:
+                f.write(json.dumps(_record_to_dict(r)) + "\n")

agentlahon-0.0.1.dist-info/METADATA

@@ -0,0 +1,187 @@
+Metadata-Version: 2.4
+Name: agentlahon
+Version: 0.0.1
+Summary: Unit tests for AI agents — catch when your agent does the wrong thing, not just when it says the wrong thing.
+Author-email: Anurag Lahon <anuraglahondp@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anuraglahon16/agentcheck
+Project-URL: Repository, https://github.com/anuraglahon16/agentcheck
+Project-URL: Issues, https://github.com/anuraglahon16/agentcheck/issues
+Keywords: ai,agents,evaluation,evals,llm,testing,governance
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Provides-Extra: judge
+Requires-Dist: openai>=1; extra == "judge"
+Provides-Extra: webapp
+Requires-Dist: flask>=3; extra == "webapp"
+Dynamic: license-file
+
+# agentlahon
+
+### Unit tests for AI agents — catch when your agent *does* the wrong thing, not just when it *says* the wrong thing.
+
+![tests](https://img.shields.io/badge/tests-passing-2fbf71) ![python](https://img.shields.io/badge/python-3.9%2B-blue) ![license](https://img.shields.io/badge/license-MIT-black) ![deps](https://img.shields.io/badge/core%20deps-zero-black)
+
+> **Status:** v0.0.1, experimental — APIs may change. Core is zero-dependency. Feedback and first users very welcome.
+
+Text-only evals grade what an agent *says*. But agents take *actions* — call tools, write to databases, move money. The dangerous failure is when the reply looks perfect while the actions are wrong. **agentlahon grades both**, and maps every check to a governance control so a green suite doubles as an audit-ready **AI assurance report**.
+
+```
+FAIL  decline-only-noncarried
+    ✓ reply mentions 'unable'          found                          ← SAID the right thing
+    ✓ never records a sale             no such action ✓
+    ✗ never restocks (money moves)     took 1×: reorder Invitation-cards ×9000   ← DID the wrong thing
+```
+
+> That failure is invisible to every text-based eval. Only checking the agent's **actions** catches it. This is the beachhead: single-turn LLM eval is crowded — **agent** eval (multi-step, tool-calling, side-effecting) is wide open.
+
+```python
+Scenario("decline-only", "5000 flyers, 10000 tickets", checks=[
+    expect.output_contains("unable"),      # say-level
+    expect.no_action("reorder_item",       # do-level — the part text evals miss
+        where=lambda a: a.result.get("ordered")),   # assert on the *effect*, not just the call
+])
+```
+
+## Install
+
+```bash
+pip install agentlahon     # PyPI distribution name
+pip install -e .            # or from source
+```
+
+Core is dependency-free. Python ≥ 3.9.
+
+## Quickstart
+
+```python
+from agentlahon import Scenario, expect, evaluate, print_terminal, write_html
+
+def agent(text):              # wrap YOUR agent -> AgentRun(output, actions)
+    ...
+
+scenarios = [
+    Scenario("decline-noncarried", "5000 flyers, 10000 tickets", checks=[
+        expect.output_contains("unable"),
+        expect.no_action("reorder_item"),   # the do-level check text evals miss
+        expect.no_pii(),
+    ]),
+]
+report = evaluate(agent, scenarios)
+print_terminal(report)
+write_html(report)            # shareable assurance_report.html
+```
+
+## The evals flywheel — look at data → tag → generate assertions
+
+The hard part of evals isn't running assertions, it's *knowing what to assert*. agentlahon logs real runs, lets you review and tag failures (open coding), rolls them into a failure taxonomy, and turns a tagged-bad trace into the assertion that would have caught it — the Hamel Husain / Shreya Shankar error-analysis loop, for agent actions.
+
+```bash
+agentlahon run suite.py --log traces.jsonl   # 1. log real runs
+agentlahon review traces.jsonl               # 2. page through, tag failures
+agentlahon analyze traces.jsonl              # 3. failure taxonomy (what dominates)
+agentlahon suggest traces.jsonl t0003        # 4. trace -> ready-to-paste assertion
+```
+
+```
+[t0003] input: 5000 flyers, 2000 posters, 10000 tickets
+      trace:  1. reorder_item(Invitation cards, 9000) → ordered=True
+      suggested assertions:
+        expect.no_action("reorder_item", where=lambda a: (a.result or {}).get("ordered"))
+            # reorder_item took effect — guard against it when it should not fire
+```
+
+## LLM-as-judge — for subjective checks, aligned before you trust it
+
+Code assertions can't judge "is this reply faithful / on-policy / correct for our domain?" — that needs an LLM judge. agentlahon does it the rigorous way: **binary** verdicts, an **optional domain reference** to grade against, and an **alignment** step that scores the judge against *your* human labels. An unaligned judge is worse than none.
+
+```python
+from agentlahon import expect, openai_complete, llm_judge, align, print_alignment
+
+complete = openai_complete(model="gpt-4o-mini")          # pluggable; pip install "agentlahon[judge]"
+
+check = expect.judge("Does the reply stay within our refund policy?",
+                     complete=complete,
+                     reference=open("refund_policy.md").read())   # optional domain doc
+
+# Don't trust the judge until it agrees with you:
+labeled = [("we'll refund within 30 days", True), ("sure, full refund anytime", False), ...]
+print_alignment(align(llm_judge("within policy?", complete), labeled))
+#  accuracy 92%  TPR 95%  TNR 88%  κ 0.83  -> TRUSTWORTHY
+```
+
+A judge that rubber-stamps everything scores **TNR 0% → NEEDS WORK** — caught before it hides real failures.
+
+## Traces — *what* went wrong, not just *that* it did
+
+When a check fails, agentlahon prints the agent's action trace and points at the exact step:
+
+```
+✗ never restocks (money moves)   took 1×: [Invitation cards ×9000]
+trace (what the agent did):
+  · 1. tool_reorder_item(Flyers, 5050)          → ordered=False   (refused, harmless)
+  · 2. tool_reorder_item(Poster paper, 2050)    → ordered=False   (refused, harmless)
+  ✗ 3. tool_reorder_item(Invitation cards, 9000)→ ordered=True    ← never restocks (money moves)
+```
+
+The failing check is linked to the offending step (`ScenarioResult.blame()`), so you go from red to root cause instantly. Same trace renders in the HTML report.
+
+## CLI (drop into CI)
+
+A suite file defines `scenarios` and `agent`; the CLI exits non-zero on findings:
+
+```bash
+agentlahon run examples/suite.py --html report.html
+```
+
+## Run the examples (no API key)
+
+```bash
+python examples/run_demo.py       # synthetic agent with a planted bug
+python examples/run_beavers.py    # points at a REAL pydantic-ai agent, catches a real regression
+pytest                            # 6 core tests
+```
+
+## Layout
+
+```
+src/agentlahon/   core · checks · adapters · report · cli
+examples/         run_demo · run_beavers · suite
+tests/            test_core
+```
+
+## Checks
+
+| Family | Checks | Control (NIST AI RMF) |
+|---|---|---|
+| say-level | `output_contains`, `output_absent` | MEASURE-2.3 Task performance |
+| **do-level** | `no_action`, `action_taken`, `max_actions`, `actions_only_on` | MANAGE-2.1 Action safety |
+| privacy | `no_pii` | MEASURE-2.10 Privacy |
+| transparency | `no_internal_leak` | MEASURE-2.9 Transparency |
+| faithfulness | `faithful(judge)` — plug in LLM-as-judge | MEASURE-2.5 Validity |
+
+## Roadmap (v0 → product)
+
+- [x] Function-capture adapter (auto-records real tool calls **and effects**)
+- [x] CI integration (`agentlahon run` exits non-zero on findings)
+- [ ] Native adapters for OpenAI/Anthropic tool-calls & LangGraph traces
+- [ ] LLM-as-judge faithfulness + bias checks
+- [ ] Regression mode: diff a run against a saved baseline on model/prompt change
+- [ ] Hosted dashboard + shareable report links (the paid layer)
+
+Status: **v0.0.1** — installable package, CLI, effect-level checks, terminal + HTML
+report, tests, and a working run against a real pydantic-ai agent.

agentlahon-0.0.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+agentlahon/__init__.py,sha256=6U_b00npJ-JDRG2jsV0kcY3ftZEpCkyIsX2ZGk2ojHQ,1320
+agentlahon/adapters.py,sha256=cEnJirnnyBel9GOzb5KQlWLA3Mf9UBjENBswsLUSFsE,2565
+agentlahon/analysis.py,sha256=NZaR6gd-4tU5iD1t95JcplCEJpG9bXcR0JXJ1BLi8UE,2870
+agentlahon/checks.py,sha256=UWSAgTTFEa1UAs4QIwCpA-izaLE4Bq-nWrvf5JQL-8g,6171
+agentlahon/cli.py,sha256=OJKFjIGJEAc_fxRiu3qHhVJH0qCXaoGwfasNoLTg3MI,5651
+agentlahon/core.py,sha256=Imnzo6jX8ogknLOYC7UbgI-JACqbWYsuVcgaqj4kw2o,4933
+agentlahon/judge.py,sha256=KCpya4cXZDGQUkltjBe6mIhlIVRPl733WLL2vuv_4_g,5655
+agentlahon/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+agentlahon/report.py,sha256=WdDLSdCV8a8VYOL3nvzs7rfqTTgbIxLLvr3rOlijZHw,7804
+agentlahon/store.py,sha256=DWSqq4Hp8mgSdVig31batvd0c6NPElwsszG_0pEFDJo,3305
+agentlahon-0.0.1.dist-info/licenses/LICENSE,sha256=3MDlg5-4nqpRTZIameTXgo5kxGeWsXdup3HWiNj-qFQ,1080
+agentlahon-0.0.1.dist-info/METADATA,sha256=5Bo8OR0hDHutQTqVQpod0xcaN5PsRH4vAigz-WGvBkk,8416
+agentlahon-0.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+agentlahon-0.0.1.dist-info/entry_points.txt,sha256=H4H2Vnanb_vzEgd6nv8t2ikJycB7h7noI_cypcYnuVM,51
+agentlahon-0.0.1.dist-info/top_level.txt,sha256=tlm_aI7A2oTai1ijcImYweFx9NLEMdRMDkXSpHZcrh8,11
+agentlahon-0.0.1.dist-info/RECORD,,

agentlahon-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

agentlahon-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+agentlahon = agentlahon.cli:main

agentlahon-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 agentcheck contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agentlahon-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+agentlahon