nabit 0.3.0__py3-none-any.whl

nabit/__init__.py

@@ -0,0 +1,57 @@
+"""nabit — did it actually work?
+
+A dead-simple verification layer for LLM agents. Your agent says it's done;
+nabit checks the real system state and tells you whether that was true — and
+can re-run the action to fix it when it wasn't.
+
+Basic usage:
+
+    from nabit import verify
+    from nabit.checks import has_keys
+
+    @verify(lambda result, ctx: db.exists("customers", result["id"]),
+            retries=2)                       # self-heal: re-run on failure
+    def create_customer(name):
+        # ... agent / tool does the work, claims success ...
+        return {"id": 42, "status": "created"}
+
+If the post-condition returns False, nabit records a failed verification,
+re-runs the action up to `retries` times, and (depending on mode) warns or
+raises — so a lying "success" can't slip through silently.
+
+What sets it apart: self-heal retries, run-ID correlation, a composable check
+library (nabit.checks), and pluggable sinks — all with zero dependencies.
+"""
+
+from .core import (
+    verify,
+    run,
+    summary,
+    VerificationError,
+    VerificationResult,
+    get_results,
+    clear_results,
+    add_sink,
+    clear_sinks,
+    Mode,
+)
+from .sinks import jsonl_sink
+
+from . import checks
+
+__all__ = [
+    "verify",
+    "run",
+    "summary",
+    "VerificationError",
+    "VerificationResult",
+    "get_results",
+    "clear_results",
+    "add_sink",
+    "clear_sinks",
+    "jsonl_sink",
+    "checks",
+    "Mode",
+]
+
+__version__ = "0.3.0"

nabit/adapters/__init__.py

@@ -0,0 +1,10 @@
+"""Optional framework adapters for nabit.
+
+Each adapter lives in its own module and lazily imports its framework, so the
+core `nabit` package stays dependency-free. Import the one you need:
+
+    from nabit.adapters.langgraph import NabitCallback, verify_node
+
+Nothing here is imported by `nabit/__init__.py` — installing nabit never pulls
+in LangGraph/LangChain.
+"""

nabit/adapters/langgraph.py

@@ -0,0 +1,176 @@
+"""LangGraph / LangChain adapter for nabit.
+
+Two ways to plug in, depending on how much you want:
+
+1. NabitCallback — a passive callback handler. Add it once to your graph and it
+   verifies every tool result against the checks you register. Zero restructure.
+   Detection only (a callback can't re-run a tool), but you still get recording,
+   run-id grouping, sinks, and summaries.
+
+       from nabit.adapters.langgraph import NabitCallback
+       from nabit.checks import truthy, has_keys
+
+       cb = NabitCallback(checks={
+           "create_customer": has_keys("id"),
+           "search_hotels": truthy(),          # catches the empty-list silent fail
+       })
+       graph.invoke(state, config={"callbacks": [cb]})
+       print(cb.summary())
+
+2. verify_node — wrap a single graph node to get the FULL nabit treatment,
+   including self-heal retries (a node is just a function, so it can be re-run):
+
+       from nabit.adapters.langgraph import verify_node
+       from nabit.checks import predicate
+
+       builder.add_node("book", verify_node(
+           book_node,
+           predicate(lambda update, state: db.has_booking(update["booking_id"])),
+           retries=2,
+       ))
+
+This module lazily imports LangChain — if it isn't installed, `verify_node` and
+the plain decorator still work; only the live callback hookup needs it.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Callable, Dict, Optional, Union
+
+from ..core import (
+    Mode,
+    VerificationResult,
+    _current_run,
+    _emit,
+    verify as _verify,
+)
+
+try:  # lazy / optional — nabit never hard-depends on LangChain
+    from langchain_core.callbacks import BaseCallbackHandler
+    _HAS_LC = True
+except Exception:  # noqa: BLE001
+    BaseCallbackHandler = object  # type: ignore[assignment,misc]
+    _HAS_LC = False
+
+
+PostCondition = Callable[[Any, dict], bool]
+
+
+def _extract_output(output: Any) -> Any:
+    """LangChain tool output may be a ToolMessage, a str, or a raw value.
+    Pull out the payload we should verify."""
+    if hasattr(output, "content"):
+        return output.content
+    return output
+
+
+class NabitCallback(BaseCallbackHandler):
+    """A LangChain/LangGraph callback handler that verifies tool results.
+
+    Register a `checks` mapping of ``tool_name -> post_condition``. After each
+    tool runs, the matching post-condition is checked against the tool's output;
+    results are recorded through nabit's normal machinery (so get_results(),
+    sinks, and run ids all work). Provide `default_check` to verify every tool.
+
+    Args:
+        checks: {tool_name: (output, ctx) -> bool}. ctx carries {"tool", "inputs"}.
+        default_check: applied to any tool without a specific entry.
+        mode: Mode.WARN (default), LOG, SILENT, or RAISE. Note RAISE inside a
+            callback may be swallowed by the framework — prefer WARN/SILENT and
+            inspect summary()/get_results().
+        run_id: optional id stamped on every result for grouping.
+    """
+
+    def __init__(
+        self,
+        checks: Optional[Dict[str, PostCondition]] = None,
+        *,
+        default_check: Optional[PostCondition] = None,
+        mode: Union[Mode, str] = Mode.WARN,
+        run_id: Optional[str] = None,
+    ) -> None:
+        self.checks = checks or {}
+        self.default_check = default_check
+        self.mode = Mode(mode)
+        self.run_id = run_id
+        self._names: Dict[str, str] = {}
+        self._starts: Dict[str, float] = {}
+
+    # -- LangChain callback hooks --------------------------------------------
+    def on_tool_start(self, serialized, input_str, *, run_id=None, **kwargs):  # noqa: ANN001
+        name = None
+        if isinstance(serialized, dict):
+            name = serialized.get("name")
+        name = name or kwargs.get("name") or "tool"
+        key = str(run_id)
+        self._names[key] = name
+        self._starts[key] = time.perf_counter()
+        # stash inputs for the post-condition context
+        self._names[key + ":inputs"] = kwargs.get("inputs") or input_str  # type: ignore[assignment]
+
+    def on_tool_end(self, output, *, run_id=None, **kwargs):  # noqa: ANN001
+        key = str(run_id)
+        name = self._names.pop(key, "tool")
+        inputs = self._names.pop(key + ":inputs", None)
+        start = self._starts.pop(key, None)
+        check = self.checks.get(name, self.default_check)
+        if check is None:
+            return  # nothing registered for this tool — ignore it
+        self._run_check(name, check, _extract_output(output), inputs, start)
+
+    # -- shared logic (framework-independent, so it's unit-testable) ----------
+    def _run_check(self, name: str, check: PostCondition, output: Any,
+                   inputs: Any, start: Optional[float]) -> None:
+        ctx = {"tool": name, "inputs": inputs}
+        try:
+            passed = bool(check(output, ctx))
+            err = None
+        except Exception as exc:  # noqa: BLE001
+            passed = False
+            err = f"postcondition raised: {exc}"
+        dur = (time.perf_counter() - start) * 1000 if start else 0.0
+        _emit(
+            VerificationResult(
+                name=name, passed=passed, duration_ms=dur, result=output,
+                error=err, attempts=1,
+                run_id=self.run_id or _current_run.get(),
+            ),
+            self.mode,
+        )
+
+    def summary(self) -> dict:
+        """Pass/fail summary scoped to this handler's run_id (if set)."""
+        from ..core import summary as _summary
+
+        return _summary(run_id=self.run_id) if self.run_id else _summary()
+
+
+def verify_node(
+    node: Callable,
+    postcondition: PostCondition,
+    *,
+    mode: Union[Mode, str] = Mode.WARN,
+    name: Optional[str] = None,
+    retries: int = 0,
+    backoff: float = 0.0,
+    on_retry: Optional[Callable] = None,
+):
+    """Wrap a LangGraph node function with nabit verification + self-heal.
+
+    A node is ``(state) -> state_update``. The post-condition receives
+    ``(state_update, ctx)`` where ctx holds the bound node args (so ctx["state"]
+    is the incoming state). Because a node is a plain function, this path gets
+    the full retry/self-heal behavior that the passive callback can't offer.
+    """
+    return _verify(
+        postcondition,
+        mode=mode,
+        name=name or getattr(node, "__name__", "node"),
+        retries=retries,
+        backoff=backoff,
+        on_retry=on_retry,
+    )(node)
+
+
+__all__ = ["NabitCallback", "verify_node", "_HAS_LC"]

nabit/checks.py

@@ -0,0 +1,198 @@
+"""Composable post-condition builders.
+
+Most verifications are the same handful of shapes: "the result has these keys",
+"this file now exists", "this URL returns 2xx", "this row is in the DB". Instead
+of hand-writing a lambda every time (what every other verifier makes you do),
+compose these.
+
+Every builder returns a post-condition `(result, context) -> bool` that plugs
+straight into @verify:
+
+    from nabit import verify
+    from nabit.checks import has_keys, all_of, file_exists
+
+    @verify(all_of(has_keys("id", "status"), lambda r, c: r["status"] == "created"))
+    def create(...): ...
+
+Zero dependencies. `http_ok` uses only the stdlib (urllib).
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Mapping
+
+PostCondition = Callable[[Any, dict], bool]
+
+
+# --- combinators ------------------------------------------------------------
+def all_of(*checks: PostCondition) -> PostCondition:
+    """Pass only if every check passes (logical AND)."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        return all(bool(c(result, ctx)) for c in checks)
+    return _pc
+
+
+def any_of(*checks: PostCondition) -> PostCondition:
+    """Pass if any check passes (logical OR)."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        return any(bool(c(result, ctx)) for c in checks)
+    return _pc
+
+
+def not_(check: PostCondition) -> PostCondition:
+    """Invert a check."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        return not bool(check(result, ctx))
+    return _pc
+
+
+# --- result-shape checks ----------------------------------------------------
+def has_keys(*keys: str) -> PostCondition:
+    """Result is a mapping containing all of `keys`."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        if not isinstance(result, Mapping):
+            return False
+        return all(k in result for k in keys)
+    return _pc
+
+
+def equals(expected: Any) -> PostCondition:
+    """Result equals `expected`."""
+    return lambda result, ctx: result == expected
+
+
+def field_equals(key: str, expected: Any) -> PostCondition:
+    """Result is a mapping where result[key] == expected."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        return isinstance(result, Mapping) and result.get(key) == expected
+    return _pc
+
+
+def truthy(key: str | None = None) -> PostCondition:
+    """Result (or result[key]) is truthy. Catches empty lists/strings/None —
+    the classic 'returned [] instead of failing' silent failure."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        if key is None:
+            return bool(result)
+        return isinstance(result, Mapping) and bool(result.get(key))
+    return _pc
+
+
+def non_empty() -> PostCondition:
+    """Result is non-empty (len > 0). Catches truncated/empty responses."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        try:
+            return len(result) > 0
+        except TypeError:
+            return result is not None
+    return _pc
+
+
+def _pluck(result: Any, key: Optional[str]) -> Any:
+    """Get result[key] if key given and result is a mapping/attr, else result."""
+    if key is None:
+        return result
+    if isinstance(result, Mapping):
+        return result.get(key)
+    return getattr(result, key, None)
+
+
+def min_length(n: int, key: Optional[str] = None) -> PostCondition:
+    """Result (or result[key]) has len >= n. Catches the 'evidence is just a
+    URL' / truncated-output silent failure (e.g. an agent's finding must carry
+    >100 chars of real proof, not a one-line claim)."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        val = _pluck(result, key)
+        try:
+            return len(val) >= n
+        except TypeError:
+            return False
+    return _pc
+
+
+def contains(needle: Any, key: Optional[str] = None) -> PostCondition:
+    """`needle` is contained in result (or result[key]). Works for substrings,
+    list membership, dict keys — anything supporting `in`. Catches 'the proof
+    doesn't actually mention the target host' style lies."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        val = _pluck(result, key)
+        try:
+            return needle in val
+        except TypeError:
+            return False
+    return _pc
+
+
+def matches(pattern: str, key: Optional[str] = None) -> PostCondition:
+    """A regex search succeeds against result (or result[key]), coerced to str.
+    Use for 'evidence must contain a hostname / HTTP status / timestamp'."""
+    import re
+
+    rx = re.compile(pattern)
+
+    def _pc(result: Any, ctx: dict) -> bool:
+        val = _pluck(result, key)
+        return val is not None and bool(rx.search(str(val)))
+    return _pc
+
+
+def in_range(lo: float, hi: float, key: Optional[str] = None) -> PostCondition:
+    """Numeric result (or result[key]) is within [lo, hi]. Catches values that
+    are out of spec (e.g. an LLM judge returning 0s but marking 'passed')."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        val = _pluck(result, key)
+        try:
+            return lo <= val <= hi
+        except TypeError:
+            return False
+    return _pc
+
+
+# --- real-world side-effect checks (the whole point of nabit) ----------------
+def file_exists(path_or_fn: str | Callable[[Any, dict], str]) -> PostCondition:
+    """A file exists on disk. `path_or_fn` is a literal path or a callable
+    (result, ctx) -> path, so you can derive the path from the agent's output."""
+    def _pc(result: Any, ctx: dict) -> bool:
+        path = path_or_fn(result, ctx) if callable(path_or_fn) else path_or_fn
+        return bool(path) and os.path.exists(path)
+    return _pc
+
+
+def file_fresh(path_or_fn: str | Callable[[Any, dict], str], max_age_s: float) -> PostCondition:
+    """A file exists AND was modified within the last `max_age_s` seconds.
+    Catches the 'cron job exited 0 but wrote nothing new' failure."""
+    import time
+
+    def _pc(result: Any, ctx: dict) -> bool:
+        path = path_or_fn(result, ctx) if callable(path_or_fn) else path_or_fn
+        if not path or not os.path.exists(path):
+            return False
+        return (time.time() - os.path.getmtime(path)) <= max_age_s
+    return _pc
+
+
+def predicate(fn: Callable[[Any, dict], bool]) -> PostCondition:
+    """Wrap an arbitrary check against real state — a DB lookup, API call, etc.
+    Just sugar for readability / composition:
+
+        predicate(lambda r, c: db.exists("customers", r["id"]))
+    """
+    return lambda result, ctx: bool(fn(result, ctx))
+
+
+def http_ok(url_or_fn: str | Callable[[Any, dict], str], timeout: float = 5.0) -> PostCondition:
+    """An HTTP GET to the URL returns a 2xx status. Stdlib only (urllib).
+    `url_or_fn` is a literal URL or (result, ctx) -> url."""
+    from urllib.request import urlopen
+
+    def _pc(result: Any, ctx: dict) -> bool:
+        url = url_or_fn(result, ctx) if callable(url_or_fn) else url_or_fn
+        if not url:
+            return False
+        try:
+            with urlopen(url, timeout=timeout) as resp:  # noqa: S310 — caller-supplied URL
+                return 200 <= resp.status < 300
+        except Exception:  # noqa: BLE001
+            return False
+    return _pc

nabit/core.py

@@ -0,0 +1,343 @@
+"""Core verification primitives for nabit.
+
+The whole idea: an LLM agent (or any tool it calls) returns a result that
+*claims* something happened. We don't trust the claim. We run a read-only
+post-condition against the actual system state and record whether reality
+matched the claim.
+
+What sets nabit apart from other verifiers:
+  * self-heal — on a failed check it can re-run the action (optionally with
+    feedback), so it corrects silent failures instead of just reporting them.
+  * run-ID correlation — scope a batch of actions under one run so their
+    verifications group together (the "nothing shares a run id" problem).
+  * pluggable sinks — tee every result to JSONL / OpenTelemetry / your logger
+    without nabit taking on a single runtime dependency.
+
+Zero dependencies. Works with sync or async functions. Framework-agnostic —
+LangGraph, LangChain, CrewAI, or a plain function all look the same here.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import contextvars
+import functools
+import inspect
+import logging
+import time
+import uuid
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import Any, Awaitable, Callable, Optional, Union
+
+logger = logging.getLogger("nabit")
+
+
+class Mode(str, Enum):
+    """What to do when a post-condition fails (after retries are exhausted).
+
+    RAISE  — raise VerificationError (fail loud, good for tests / CI).
+    WARN   — log at WARNING level and return the result anyway (good for prod
+             rollout: you get the signal without changing behavior).
+    LOG    — log at INFO level only.
+    SILENT — record the result but emit nothing (inspect via get_results()).
+    """
+
+    RAISE = "raise"
+    WARN = "warn"
+    LOG = "log"
+    SILENT = "silent"
+
+
+class VerificationError(AssertionError):
+    """Raised (in Mode.RAISE) when an agent's claimed outcome does not match
+    actual system state, after any retries are exhausted."""
+
+
+@dataclass
+class VerificationResult:
+    """One verification event, kept in an in-memory log for reporting."""
+
+    name: str
+    passed: bool
+    duration_ms: float
+    result: Any = None
+    error: Optional[str] = None
+    attempts: int = 1
+    run_id: Optional[str] = None
+    timestamp: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict:
+        d = asdict(self)
+        # `result` may not be JSON-serializable; stringify defensively.
+        try:
+            import json
+
+            json.dumps(d["result"])
+        except (TypeError, ValueError):
+            d["result"] = repr(d["result"])
+        return d
+
+
+# --- run-ID correlation -----------------------------------------------------
+# A context var so concurrent/async runs don't clobber each other's run id.
+_current_run: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
+    "nabit_run_id", default=None
+)
+
+
+@contextlib.contextmanager
+def run(run_id: Optional[str] = None):
+    """Scope a batch of verified actions under a single run id, so their
+    results group together.
+
+        with nabit.run() as rid:
+            create_customer(...)
+            charge_card(...)
+        print(nabit.summary(run_id=rid))
+
+    If `run_id` is omitted a short unique id is generated and yielded.
+    """
+    rid = run_id or uuid.uuid4().hex[:12]
+    token = _current_run.set(rid)
+    try:
+        yield rid
+    finally:
+        _current_run.reset(token)
+
+
+# --- results log + pluggable sinks ------------------------------------------
+_RESULTS: list[VerificationResult] = []
+_MAX_RESULTS = 1000
+_SINKS: list[Callable[[VerificationResult], None]] = []
+
+
+def add_sink(fn: Callable[[VerificationResult], None]) -> None:
+    """Register a callback invoked with every VerificationResult as it happens.
+    Use it to tee results to JSONL, OpenTelemetry, Datadog, etc. Zero deps:
+    nabit never imports your sink's backend."""
+    _SINKS.append(fn)
+
+
+def clear_sinks() -> None:
+    _SINKS.clear()
+
+
+def get_results(run_id: Optional[str] = None) -> list[VerificationResult]:
+    """Return recorded verification results (most recent last). Optionally
+    filter to a single run id."""
+    if run_id is None:
+        return list(_RESULTS)
+    return [r for r in _RESULTS if r.run_id == run_id]
+
+
+def clear_results() -> None:
+    """Clear the in-memory results log (useful between tests)."""
+    _RESULTS.clear()
+
+
+def summary(run_id: Optional[str] = None) -> dict:
+    """Aggregate pass/fail stats, optionally scoped to one run id."""
+    rs = get_results(run_id)
+    total = len(rs)
+    passed = sum(1 for r in rs if r.passed)
+    failed = total - passed
+    return {
+        "total": total,
+        "passed": passed,
+        "failed": failed,
+        "pass_rate": (passed / total) if total else 1.0,
+        "failures": [r.name for r in rs if not r.passed],
+    }
+
+
+def _record(result: VerificationResult) -> None:
+    _RESULTS.append(result)
+    if len(_RESULTS) > _MAX_RESULTS:
+        del _RESULTS[0]
+    for sink in _SINKS:
+        try:
+            sink(result)
+        except Exception:  # noqa: BLE001 — a broken sink must never break the app
+            logger.exception("nabit: sink raised; continuing")
+
+
+# A post-condition receives (result, context) and returns a truthy value for
+# "reality matches the claim". `context` is the bound call arguments, so you
+# can check the inputs against the outputs. May be sync or async.
+PostCondition = Callable[[Any, dict], Union[bool, Awaitable[bool]]]
+# A retry/feedback hook: (attempt, last_result, context) -> None. Use it to
+# nudge the agent before the next attempt (e.g. append a corrective message).
+RetryHook = Callable[[int, Any, dict], Any]
+
+
+def _build_context(func: Callable, args: tuple, kwargs: dict) -> dict:
+    """Bind call args to parameter names so post-conditions can read inputs."""
+    try:
+        bound = inspect.signature(func).bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        return dict(bound.arguments)
+    except TypeError:
+        return {"args": args, "kwargs": kwargs}
+
+
+def _emit(result: VerificationResult, mode: Mode) -> None:
+    """Record a final result and raise/log according to mode."""
+    _record(result)
+    if result.passed:
+        logger.debug("nabit: %s verified OK (%.1fms, %d attempt(s))",
+                     result.name, result.duration_ms, result.attempts)
+        return
+    msg = (f"nabit: {result.name} FAILED verification after {result.attempts} "
+           f"attempt(s) — agent claimed success, reality disagreed")
+    if result.error:
+        msg += f" ({result.error})"
+    if mode is Mode.RAISE:
+        raise VerificationError(msg)
+    elif mode is Mode.WARN:
+        logger.warning(msg)
+    elif mode is Mode.LOG:
+        logger.info(msg)
+    # SILENT: recorded only.
+
+
+def verify(
+    postcondition: PostCondition,
+    *,
+    mode: Union[Mode, str] = Mode.WARN,
+    name: Optional[str] = None,
+    on_error: bool = True,
+    retries: int = 0,
+    backoff: float = 0.0,
+    on_retry: Optional[RetryHook] = None,
+) -> Callable:
+    """Decorator: run the wrapped function, then check its claimed outcome
+    against actual system state via `postcondition`. On failure, optionally
+    self-heal by re-running the action.
+
+    Args:
+        postcondition: callable (result, context) -> bool. Truthy == reality
+            matched the claim. May be sync or async.
+        mode: what to do once retries are exhausted. See Mode. Default WARN.
+        name: label for the result log. Defaults to the function name.
+        on_error: if True (default), also run the post-condition when the
+            wrapped function raises — so "it threw but the side effect still
+            happened" (or vice versa) is handled. On the final failed attempt
+            the original exception is re-raised.
+        retries: how many times to re-run the action if verification fails
+            (0 = no self-heal, just verify once). This is the closed loop:
+            detect AND correct.
+        backoff: seconds to sleep between retries, multiplied by attempt number
+            (linear backoff). Ignored for async if 0.
+        on_retry: optional (attempt, last_result, context) -> None hook invoked
+            before each retry — use it to feed the discrepancy back to the agent.
+
+    Works transparently on both sync and async functions.
+    """
+    mode = Mode(mode)
+
+    def decorator(func: Callable) -> Callable:
+        label = name or getattr(func, "__name__", "anonymous")
+        pc_is_async = inspect.iscoroutinefunction(postcondition)
+        func_is_async = inspect.iscoroutinefunction(func)
+
+        async def _run_pc_async(result: Any, ctx: dict) -> bool:
+            out = postcondition(result, ctx)
+            if inspect.isawaitable(out):
+                out = await out
+            return bool(out)
+
+        def _run_pc_sync(result: Any, ctx: dict) -> bool:
+            if pc_is_async:
+                raise RuntimeError(
+                    f"nabit: async post-condition used on sync function '{label}'. "
+                    "Make the wrapped function async, or the post-condition sync."
+                )
+            return bool(postcondition(result, ctx))
+
+        def _finalize(result, passed, pc_err, last_exc, attempt, start):
+            """Record the final attempt, then raise/return per semantics:
+            a real exception always propagates (more informative than a
+            VerificationError); otherwise _emit handles raise/warn/log/silent."""
+            err = pc_err or (f"function raised: {last_exc}" if last_exc else None)
+            vr = VerificationResult(
+                name=label, passed=passed,
+                duration_ms=(time.perf_counter() - start) * 1000,
+                result=result, error=err, attempts=attempt,
+                run_id=_current_run.get(),
+            )
+            if last_exc is not None:
+                _record(vr)
+                if not passed and mode in (Mode.WARN, Mode.LOG):
+                    logger.warning("nabit: %s FAILED (function raised after %d "
+                                   "attempt(s)): %s", label, attempt, last_exc)
+                raise last_exc
+            _emit(vr, mode)
+            return result
+
+        if func_is_async:
+            @functools.wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                import asyncio
+
+                ctx = _build_context(func, args, kwargs)
+                start = time.perf_counter()
+                result: Any = None
+                for attempt in range(1, retries + 2):  # 1 initial + `retries`
+                    last_exc: Optional[BaseException] = None
+                    try:
+                        result = await func(*args, **kwargs)
+                    except Exception as exc:  # noqa: BLE001
+                        last_exc = exc
+                        result = None
+                        if not on_error:
+                            raise  # don't verify, just propagate
+                    try:
+                        passed = await _run_pc_async(result, ctx)
+                        pc_err = None
+                    except Exception as pc_exc:  # noqa: BLE001
+                        passed = False
+                        pc_err = f"postcondition raised: {pc_exc}"
+                    if passed or attempt == retries + 1:
+                        return _finalize(result, passed, pc_err, last_exc, attempt, start)
+                    logger.debug("nabit: %s attempt %d failed, retrying", label, attempt)
+                    if on_retry is not None:
+                        maybe = on_retry(attempt, result, ctx)
+                        if inspect.isawaitable(maybe):
+                            await maybe
+                    if backoff:
+                        await asyncio.sleep(backoff * attempt)
+
+            return async_wrapper
+
+        @functools.wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            ctx = _build_context(func, args, kwargs)
+            start = time.perf_counter()
+            result: Any = None
+            for attempt in range(1, retries + 2):
+                last_exc: Optional[BaseException] = None
+                try:
+                    result = func(*args, **kwargs)
+                except Exception as exc:  # noqa: BLE001
+                    last_exc = exc
+                    result = None
+                    if not on_error:
+                        raise
+                try:
+                    passed = _run_pc_sync(result, ctx)
+                    pc_err = None
+                except Exception as pc_exc:  # noqa: BLE001
+                    passed = False
+                    pc_err = f"postcondition raised: {pc_exc}"
+                if passed or attempt == retries + 1:
+                    return _finalize(result, passed, pc_err, last_exc, attempt, start)
+                logger.debug("nabit: %s attempt %d failed, retrying", label, attempt)
+                if on_retry is not None:
+                    on_retry(attempt, result, ctx)
+                if backoff:
+                    time.sleep(backoff * attempt)
+
+        return sync_wrapper
+
+    return decorator

nabit/sinks.py

@@ -0,0 +1,41 @@
+"""Built-in result sinks.
+
+A sink is just a callable `(VerificationResult) -> None` registered with
+`nabit.add_sink(...)`. These ship in the box; anything fancier (OpenTelemetry,
+Datadog, Kafka) is a three-line function you write and register — nabit never
+imports those backends itself, so the core stays dependency-free.
+
+OpenTelemetry example (you provide the dependency, not nabit):
+
+    from opentelemetry import trace
+    tracer = trace.get_tracer("nabit")
+
+    def otel_sink(r):
+        with tracer.start_as_current_span(f"nabit.verify.{r.name}") as span:
+            span.set_attribute("nabit.passed", r.passed)
+            span.set_attribute("nabit.attempts", r.attempts)
+            if r.run_id:
+                span.set_attribute("nabit.run_id", r.run_id)
+
+    nabit.add_sink(otel_sink)
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from typing import Callable
+
+
+def jsonl_sink(path: str) -> Callable:
+    """Return a sink that appends each VerificationResult as one JSON line to
+    `path`. Thread-safe. Register with nabit.add_sink(jsonl_sink("hits.jsonl"))."""
+    lock = threading.Lock()
+
+    def _sink(result) -> None:
+        line = json.dumps(result.to_dict(), default=str)
+        with lock:
+            with open(path, "a", encoding="utf-8") as fh:
+                fh.write(line + "\n")
+
+    return _sink

nabit-0.3.0.dist-info/METADATA

@@ -0,0 +1,280 @@
+Metadata-Version: 2.5
+Name: nabit
+Version: 0.3.0
+Summary: nab your agent's silent failures — verify LLM-agent outcomes against real system state, with self-heal. Zero dependencies.
+Project-URL: Homepage, https://jakegarnier.com/agent-reliability-kit
+Project-URL: Source, https://github.com/jake-garnier/nabit
+Author-email: Jake Garnier <jakegarnier@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Jake Garnier
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agents,ai,langchain,langgraph,llm,observability,reliability,verification
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: langchain-core>=0.2; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Provides-Extra: langgraph
+Requires-Dist: langchain-core>=0.2; extra == 'langgraph'
+Description-Content-Type: text/markdown
+
+# nabit — *nab your agent's silent failures*
+
+Your LLM agent said it created the customer. Your database says otherwise. You
+found out three days later from a support ticket.
+
+**nabit** is a dead-simple verification layer for LLM agents. Your agent claims
+it's done; `nabit` checks the *real system state*, tells you whether that was
+true, and — if it wasn't — **re-runs the action to fix it.** One decorator.
+Zero dependencies. Sync or async.
+
+```python
+from nabit import verify
+
+@verify(lambda result, ctx: db.exists("customers", result["id"]),
+        retries=2)                      # self-heal: re-run on failure
+def create_customer(name):
+    # agent / tool does the work and claims success
+    return {"id": 42, "status": "created"}
+```
+
+If the agent returns `{"status": "created"}` but the row isn't in the database,
+`nabit` catches the lie instead of letting a green dashboard hide it — then
+retries the action up to `retries` times before giving up.
+
+## How it's different
+
+Most tools in this space either detect problems without fixing them, only check
+the *text the model produced* (not whether the real action happened), or make you
+stand up a backend to do it. nabit:
+
+- **checks real side effects**, not output schema (vs Guardrails AI / Instructor)
+- **closes the loop** — self-heal retries, not just detection (vs Drift / trace viewers)
+- **verifies inline at runtime**, not in a postmortem (vs agent-coroner)
+- **has zero dependencies and no backend** — it's a decorator, not a platform (vs COGEXT)
+
+## Why this exists
+
+Trace viewers (Langfuse, LangSmith, Helicone) answer *"what did the agent do?"*
+really well. They don't answer *"was what it did actually correct?"*
+
+The expensive failures are **semantic**, not technical: no exception thrown, the
+tool call succeeded, and the output was still wrong. Those look identical to a
+success at the log level. `nabit` is the layer that checks the claim against
+reality.
+
+## Install
+
+```bash
+pip install nabit
+```
+
+## Usage
+
+### Post-conditions check reality, not the agent's word
+
+A post-condition is `(result, context) -> bool`. `result` is what the function
+returned; `context` is the bound call arguments (so you can compare inputs to
+outputs). Return truthy if reality matches the claim.
+
+```python
+from nabit import verify
+
+@verify(lambda result, ctx: ticket_store.is_closed(result["ticket_id"]))
+def close_ticket(ticket_id):
+    agent.act(f"close ticket {ticket_id}")
+    return {"ticket_id": ticket_id, "status": "closed"}
+```
+
+### Modes — tune how loud failures are
+
+```python
+from nabit import verify, Mode
+
+@verify(check, mode=Mode.WARN)    # log a warning, keep running  (default — safe for prod)
+@verify(check, mode=Mode.RAISE)   # raise VerificationError      (great for tests / CI)
+@verify(check, mode=Mode.LOG)     # info-level log only
+@verify(check, mode=Mode.SILENT)  # record only; inspect later
+```
+
+### Self-heal — re-run the action when verification fails
+
+```python
+def feedback(attempt, last_result, ctx):
+    # optional: nudge the agent before the next attempt
+    log.warning("verification failed on attempt %d, retrying", attempt)
+
+@verify(check, retries=3, backoff=0.5, on_retry=feedback)
+def book_flight(req):
+    ...
+```
+
+`retries` re-runs the whole action up to N times until the post-condition
+passes; `backoff` adds linear delay between attempts; `on_retry` lets you feed
+the discrepancy back to the agent. Detection *and* correction, in one decorator.
+
+### Composable checks (no hand-written lambdas)
+
+```python
+from nabit import verify
+from nabit.checks import (all_of, has_keys, field_equals, file_fresh, http_ok,
+                         truthy, min_length, contains, matches, in_range)
+
+@verify(all_of(has_keys("id", "status"), field_equals("status", "created")))
+def create(...): ...
+
+@verify(file_fresh(lambda r, c: r["path"], max_age_s=60))   # cron wrote a fresh file?
+def nightly_report(): ...
+
+@verify(http_ok(lambda r, c: r["url"]))                     # deployed URL is live?
+def deploy(): ...
+
+@verify(truthy())                                           # not [] / "" / None
+def search(...): ...
+
+# an agent's "reportable" finding must carry real proof, not a one-line claim
+@verify(all_of(min_length(100, key="evidence"), matches(r"https?://", key="evidence")))
+def finish_task(finding): ...
+
+@verify(in_range(1, 10, key="score"))                       # LLM judge score in spec
+def judge(...): ...
+```
+
+Full check list: `all_of` / `any_of` / `not_`, `has_keys`, `equals`,
+`field_equals`, `truthy`, `non_empty`, `min_length`, `contains`, `matches`,
+`in_range`, `file_exists`, `file_fresh`, `http_ok`, `predicate`.
+
+### Group verifications under a run id
+
+```python
+from nabit import run, summary
+
+with run("signup-flow") as rid:
+    create_customer(...)
+    charge_card(...)
+
+print(summary(run_id=rid))
+# {'total': 2, 'passed': 1, 'failed': 1, 'pass_rate': 0.5, 'failures': ['charge_card']}
+```
+
+Solves the "nothing shares a run id" problem — all the checks for one logical
+task carry the same id.
+
+### Tee results anywhere (pluggable sinks, still zero-dep)
+
+```python
+import nabit
+from nabit import jsonl_sink
+
+nabit.add_sink(jsonl_sink("verifications.jsonl"))   # built-in
+
+def otel_sink(r):                                   # or your own, 3 lines
+    span.set_attribute("nabit.passed", r.passed)
+nabit.add_sink(otel_sink)
+```
+
+### LangGraph / LangChain
+
+nabit works with any framework via the decorator, but LangGraph users get a
+first-class adapter (lazily imported — installing nabit never pulls in
+LangChain). Two options:
+
+**Passive callback** — add it once, verify every tool result, no restructure:
+
+```python
+from nabit.adapters.langgraph import NabitCallback
+from nabit.checks import has_keys, truthy
+
+cb = NabitCallback(checks={
+    "create_customer": has_keys("id"),
+    "search_hotels": truthy(),        # catches the empty-list silent failure
+})
+graph.invoke(state, config={"callbacks": [cb]})
+print(cb.summary())
+```
+
+**Node wrapper** — wrap a node to get full self-heal (a node is just a function,
+so it can be re-run):
+
+```python
+from nabit.adapters.langgraph import verify_node
+from nabit.checks import predicate
+
+builder.add_node("book", verify_node(
+    book_node,
+    predicate(lambda update, ctx: db.has_booking(update["booking_id"])),
+    retries=2,
+))
+```
+
+### Async works the same way
+
+```python
+@verify(pc, retries=2)                    # async funcs + async post-conditions
+async def create_order(cart):
+    ...
+```
+
+### Inspect what happened
+
+```python
+from nabit import get_results
+
+for r in get_results():
+    print(r.name, "PASS" if r.passed else "FAIL",
+          f"{r.duration_ms:.0f}ms", f"attempts={r.attempts}", r.error)
+```
+
+### Also catches "it threw but the side effect still happened"
+
+By default (`on_error=True`) the post-condition runs even when the wrapped
+function raises — so a tool that errors *after* mutating state (or succeeds in
+reality despite throwing) still gets verified. The original exception is
+re-raised after recording.
+
+## Scope
+
+`nabit` is the **outcome verifier**: it answers *"did this specific action
+actually happen?"* and corrects it when it didn't. It deliberately does **not**
+try to be an observability platform. For *fleet-wide* behavioral monitoring —
+real-time degradation detection across many runs (step-count blowups, token
+spikes, slow drift over hundreds of runs), trajectory snapshot diffing, and a
+dashboard — see **[The Production Agent Reliability Kit](https://jakegarnier.com/agent-reliability-kit)**,
+built on published research
+([SENTINEL](https://github.com/jake-garnier/sentinel), self-supervised anomaly
+detection for LLM agents).
+
+Rule of thumb: use `nabit` to verify *one action's* real effect inline; reach
+for the Kit when you need to watch *patterns across runs* over time.
+
+## License
+
+MIT — see [LICENSE](LICENSE). Use it anywhere, including commercially.
+
+---
+
+Built by [Jake Garnier](https://jakegarnier.com).

nabit-0.3.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+nabit/__init__.py,sha256=D8xszU7YpoGstTrqhW6ZsqXBJSWplEgESqFpA1gDvG0,1458
+nabit/checks.py,sha256=2T2ITrojCd0NMsbonlXt7qdDhooSp6zEN4PbW33EU40,7070
+nabit/core.py,sha256=sC5ClyIUH28L7JVFjjR-S2zyTSt6eQOYy0Ag-N-AjUA,13070
+nabit/sinks.py,sha256=qMTdUtLf7-iuRkwL7AZziUNgF0m5TcBehL9vc1m4HQk,1351
+nabit/adapters/__init__.py,sha256=KvQWOT8T84zKPC6a2E992uJeQ6cYyuNZrd62RRmDFAE,369
+nabit/adapters/langgraph.py,sha256=2yD7DqEp0vG0Ji_3_HWae0rRZf6uRWqLSOdLyHnZlPQ,6483
+nabit-0.3.0.dist-info/METADATA,sha256=zne-jGZ_7CjBECrw6THfE4QbnxXW5QDbD81xPnuGOuQ,10220
+nabit-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+nabit-0.3.0.dist-info/licenses/LICENSE,sha256=P-wcQkV8adB4cgZ4km_ys9Opnln6bRucGHHIcR_N8cM,1069
+nabit-0.3.0.dist-info/RECORD,,

nabit-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

nabit-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jake Garnier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.