probity-effects 0.1.0__py3-none-any.whl

probity_effects/__init__.py

@@ -0,0 +1,33 @@
+"""Keep one effect per agent tool call when its reply is lost.
+
+The guard reconciles with the service that owns the effect before anything is sent again, over a
+journal that is also the operation's uncertain-effect/v1 record.
+"""
+
+from probity_effects.guard import EffectGuard, EffectOutcomeUnknown, Outcome
+from probity_effects.journal import Journal
+from probity_effects.owner import (
+    EffectCall,
+    EffectOwner,
+    Lookup,
+    NoEffect,
+    Reply,
+    classify_failure,
+)
+from probity_effects.record import CONTRACT_ID, args_digest, derive_key
+
+__all__ = [
+    "CONTRACT_ID",
+    "EffectCall",
+    "EffectGuard",
+    "EffectOutcomeUnknown",
+    "EffectOwner",
+    "Journal",
+    "Lookup",
+    "NoEffect",
+    "Outcome",
+    "Reply",
+    "args_digest",
+    "classify_failure",
+    "derive_key",
+]

probity_effects/guard.py

@@ -0,0 +1,254 @@
+"""Reconcile before resend: one policy, driven synchronously or asynchronously."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import threading
+import time
+from collections.abc import Callable, Generator, Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+from probity_effects.journal import Attempt, Journal
+from probity_effects.owner import EffectCall, EffectOwner, Lookup, Reply
+from probity_effects.record import args_digest, derive_key, jsonable, parse
+
+
+@dataclass(frozen=True)
+class Outcome:
+    """What one call of the guard reports.
+
+    ``outcome`` is ``committed``, ``not_run`` or ``unknown``. ``value`` is what the send returned when
+    this call's attempt was confirmed. ``sent`` counts the dispatches this call made.
+    """
+
+    outcome: str
+    key: str
+    reason: str
+    value: Any = None
+    sent: int = 0
+    error: str | None = None
+
+    @property
+    def committed(self) -> bool:
+        return self.outcome == "committed"
+
+
+class EffectOutcomeUnknown(RuntimeError):
+    """Raised by a tool configured to stop, rather than report, when the outcome is unknown."""
+
+    def __init__(self, outcome: Outcome) -> None:
+        super().__init__(f"outcome unknown for idempotency key {outcome.key}: {outcome.reason}")
+        self.outcome = outcome
+
+
+# What the policy asks its driver to do.
+Request = tuple[str, Any]
+Plan = Generator[Request, Any, Outcome]
+
+
+class EffectGuard:
+    """Perform an operation at most once in effect, whatever retries, re-asks and resumes do."""
+
+    def __init__(self, owner: EffectOwner, journal: Journal | None = None) -> None:
+        self.owner = owner
+        self.journal = journal if journal is not None else Journal()
+        self._locks: dict[str, threading.Lock] = {}
+        self._async_locks: dict[str, asyncio.Lock] = {}
+        self._registry = threading.Lock()
+
+    def key_for(self, action: str, args: Mapping[str, Any], scope: str = "") -> str:
+        return derive_key(action, jsonable(dict(args)), scope)
+
+    def perform(self, action: str, args: Mapping[str, Any], *, send: Callable[[EffectCall], Any] | None = None,
+                key: str | None = None, scope: str = "", fencing_token: int | None = None) -> Outcome:
+        plain, digest, key = self._identify(action, args, key, scope)
+        sender = self._sender(send)
+        with self._lock(key):
+            plan = self._plan(action, key, plain, digest, fencing_token)
+            try:
+                request = next(plan)
+                while True:
+                    request = plan.send(self._serve(request, sender))
+            except StopIteration as done:
+                return done.value
+
+    async def aperform(self, action: str, args: Mapping[str, Any], *, send: Callable[[EffectCall], Any] | None = None,
+                       key: str | None = None, scope: str = "", fencing_token: int | None = None) -> Outcome:
+        plain, digest, key = self._identify(action, args, key, scope)
+        sender = self._sender(send)
+        async with self._async_lock(key):
+            plan = self._plan(action, key, plain, digest, fencing_token)
+            try:
+                request = next(plan)
+                while True:
+                    request = plan.send(await self._aserve(request, sender))
+            except StopIteration as done:
+                return done.value
+
+    # The drivers: the only code that touches the owner.
+
+    def _identify(self, action: str, args: Mapping[str, Any], key: str | None,
+                  scope: str) -> tuple[dict[str, Any], str, str]:
+        plain = jsonable(dict(args))
+        return plain, args_digest(plain), key or derive_key(action, plain, scope)
+
+    def _sender(self, send: Callable[[EffectCall], Any] | None) -> Callable[[EffectCall], Any]:
+        sender = self.owner.send or send
+        if sender is None:
+            raise TypeError("nothing performs the effect: pass send=, or give the owner a send")
+        return sender
+
+    def _lock(self, key: str) -> threading.Lock:
+        with self._registry:
+            return self._locks.setdefault(key, threading.Lock())
+
+    def _async_lock(self, key: str) -> asyncio.Lock:
+        with self._registry:
+            return self._async_locks.setdefault(key, asyncio.Lock())
+
+    def _serve(self, request: Request, sender: Callable[[EffectCall], Any]) -> Any:
+        kind, payload = request
+        if kind == "sleep":
+            time.sleep(payload)
+            return None
+        if kind == "send":
+            try:
+                return Reply.CONFIRMED, sender(payload), None
+            except Exception as error:  # noqa: BLE001 - every failure is read, none is lost
+                return self.owner.read_failure(error), None, f"{type(error).__name__}: {error}"
+        return self._ask(kind, payload, lambda result: result)
+
+    async def _aserve(self, request: Request, sender: Callable[[EffectCall], Any]) -> Any:
+        kind, payload = request
+        if kind == "sleep":
+            await asyncio.sleep(payload)
+            return None
+        if kind == "send":
+            try:
+                value = sender(payload)
+                return Reply.CONFIRMED, (await value if inspect.isawaitable(value) else value), None
+            except Exception as error:  # noqa: BLE001 - every failure is read, none is lost
+                return self.owner.read_failure(error), None, f"{type(error).__name__}: {error}"
+        try:
+            result = self._call_owner(kind, payload)
+            return await result if inspect.isawaitable(result) else result
+        except Exception:  # noqa: BLE001 - an owner that cannot answer is no answer
+            return None
+
+    def _ask(self, kind: str, payload: Any, finish: Callable[[Any], Any]) -> Any:
+        try:
+            result = self._call_owner(kind, payload)
+        except Exception:  # noqa: BLE001 - an owner that cannot answer is no answer
+            return None
+        if inspect.isawaitable(result):
+            raise TypeError(f"the owner's {kind} is asynchronous; call aperform")
+        return finish(result)
+
+    def _call_owner(self, kind: str, payload: Any) -> Any:
+        if kind == "lookup":
+            return self.owner.lookup(payload)
+        if kind == "lease":
+            return self.owner.lease() if self.owner.lease is not None else None
+        raise ValueError(f"unknown request {kind!r}")
+
+    # The policy: no I/O of its own, only requests to a driver and writes to the journal.
+
+    def _plan(self, action: str, key: str, args: dict[str, Any], digest: str, token: int | None) -> Plan:
+        journal, owner = self.journal, self.owner
+        history = journal.history(key)
+        if history.operation is not None and history.operation["argsDigest"] != digest:
+            journal.append({"event": "refused", "key": key, "argsDigest": digest,
+                            "reason": "key used with other arguments"})
+            return Outcome("not_run", key, f"idempotency key {key} already names an operation with other "
+                                           "arguments, so nothing was sent")
+        if history.operation is None:
+            journal.append({"event": "operation", "key": key, "action": action, "argsDigest": digest,
+                            "owner": owner.describe()})
+        if history.committed:
+            return self._report(key, "committed", "the owner already holds this effect, so it was not sent again")
+        pending = history.unresolved()
+        if pending is not None and not self._deduplicated(pending):
+            answer = yield from self._settle(key)
+            if answer is not False:
+                return self._settled(key, answer, "an earlier reply was lost")
+        sent = 0
+        while True:
+            history = journal.history(key)
+            if len(history.attempts) >= owner.max_sends:
+                settled = history.settled()
+                return self._report(key, settled, f"the send limit of {owner.max_sends} is reached", sent=sent)
+            if token is not None:
+                current = yield ("lease", None)
+                held = current is not None and current == token
+                journal.append({"event": "lease", "key": key, "token": current, "held": held})
+                if not held:
+                    return (yield from self._lease_lost(key, history, sent))
+            attempt = len(history.attempts) + 1
+            call = EffectCall(key=key, action=action, args=args, attempt=attempt, fencing_token=token)
+            journal.append({"event": "dispatch", "key": key, "attempt": attempt, "argsDigest": digest,
+                            **({"fencingToken": token} if token is not None else {})})
+            reply, value, error = yield ("send", call)
+            sent += 1
+            journal.append({"event": "reply", "key": key, "attempt": attempt, "reply": reply.value,
+                            **({"detail": error} if error else {})})
+            if reply is Reply.CONFIRMED:
+                return self._report(key, "committed", "the owner confirmed the effect", value=value, sent=sent)
+            if reply is Reply.NO_EFFECT:
+                return self._report(key, "not_run", f"the owner answered that the effect did not happen ({error})",
+                                    sent=sent, error=error)
+            if self._deduplicated(journal.history(key).unresolved()):
+                continue
+            answer = yield from self._settle(key)
+            if answer is not False:
+                return self._settled(key, answer, "the reply was lost", sent=sent, error=error)
+
+    def _lease_lost(self, key: str, history: Any, sent: int) -> Plan:
+        if history.unresolved() is None:
+            return self._report(key, history.settled(), "this worker no longer holds the lease, so nothing was sent",
+                                sent=sent)
+        answer = yield from self._settle(key)
+        if answer is False:
+            return self._report(key, "not_run", "the lease moved and the owner holds no effect", sent=sent)
+        return self._settled(key, answer, "the lease moved after a lost reply", sent=sent)
+
+    def _settle(self, key: str) -> Generator[Request, Any, bool | None]:
+        """Ask the owner until it answers finally: True present, False absent, None no final answer."""
+        for number in range(self.owner.lookups):
+            if number:
+                yield ("sleep", self.owner.lookup_interval)
+            answer: Lookup | None = yield ("lookup", key)
+            if answer is None:
+                self.journal.append({"event": "lookup", "key": key, "answered": False})
+                continue
+            final = bool(answer.present or answer.final)
+            self.journal.append({"event": "lookup", "key": key, "answered": True,
+                                 "result": "present" if answer.present else "absent", "final": final})
+            if answer.present:
+                return True
+            if final:
+                return False
+        return None
+
+    def _deduplicated(self, pending: Attempt | None) -> bool:
+        """Whether the owner's deduplication covers sending the same key again now."""
+        owner = self.owner
+        if pending is None or not owner.deduplicates_by_key:
+            return False
+        if owner.key_retention_seconds is None:
+            return True
+        age = (datetime.now(UTC) - parse(pending.dispatched_at)).total_seconds()
+        return age <= owner.key_retention_seconds
+
+    def _settled(self, key: str, answer: bool | None, why: str, *, sent: int = 0, error: str | None = None) -> Outcome:
+        if answer is True:
+            return self._report(key, "committed", f"{why}, and the owner holds the effect", sent=sent, error=error)
+        return self._report(key, "unknown", f"{why}, and the owner could not confirm what it holds",
+                            sent=sent, error=error)
+
+    def _report(self, key: str, outcome: str, reason: str, *, value: Any = None, sent: int = 0,
+                error: str | None = None) -> Outcome:
+        self.journal.append({"event": "outcome", "key": key, "outcome": outcome, "reason": reason})
+        return Outcome(outcome=outcome, key=key, reason=reason, value=value, sent=sent, error=error)

probity_effects/journal.py

@@ -0,0 +1,185 @@
+"""The journal: the guard's memory, and the uncertain-effect/v1 record it exports."""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from probity_effects.record import CONTRACT_ID, Clock, parse, stamp
+
+UNKNOWN_REPLIES = ("timeout", "connection_lost")
+
+
+@dataclass(frozen=True)
+class Attempt:
+    number: int
+    key: str
+    args_digest: str
+    dispatched_at: str
+    fencing_token: int | None = None
+    reply: str | None = None  # None: dispatched, and no reply was recorded
+
+    @property
+    def unknown(self) -> bool:
+        return self.reply is None or self.reply in UNKNOWN_REPLIES
+
+
+@dataclass
+class History:
+    """Everything the journal holds about one idempotency key."""
+
+    key: str
+    operation: dict[str, Any] | None = None
+    attempts: list[Attempt] = field(default_factory=list)
+    reconciliations: list[dict[str, Any]] = field(default_factory=list)
+    lookups: int = 0
+    outcome: str | None = None
+
+    @property
+    def committed(self) -> bool:
+        return (any(item.reply == "confirmed" for item in self.attempts)
+                or any(item["result"] == "present" for item in self.reconciliations))
+
+    def unresolved(self) -> Attempt | None:
+        """The earliest lost-reply attempt, unless a final absent answer came after the last one."""
+        unknown = [item for item in self.attempts if item.unknown]
+        if not unknown:
+            return None
+        last = max(parse(item.dispatched_at) for item in unknown)
+        if any(item["result"] == "absent" and item["final"] and parse(item["at"]) >= last
+               for item in self.reconciliations):
+            return None
+        return min(unknown, key=lambda item: parse(item.dispatched_at))
+
+    def settled(self) -> str:
+        """The outcome the owner's answers settle, as the record's reader computes it."""
+        if self.committed:
+            return "committed"
+        return "unknown" if self.unresolved() is not None else "not_run"
+
+
+class Journal:
+    """Append-only events per idempotency key, in memory or in a JSON-lines file.
+
+    A file journal writes each event before the guard acts on it, so a process that stops mid-call
+    leaves the dispatch behind, and the next process that opens the file reconciles it before sending
+    again. Pass ``fsync=True`` to force every line to disk.
+    """
+
+    def __init__(self, path: str | os.PathLike[str] | None = None, *, fsync: bool = False) -> None:
+        self._path = Path(path) if path is not None else None
+        self._fsync = fsync
+        self._events: list[dict[str, Any]] = []
+        self._lock = threading.RLock()
+        self.clock = Clock()
+        if self._path is not None and self._path.exists():
+            self._load(self._path)
+
+    @classmethod
+    def open(cls, path: str | os.PathLike[str], *, fsync: bool = False) -> Journal:
+        return cls(path, fsync=fsync)
+
+    @property
+    def path(self) -> Path | None:
+        return self._path
+
+    def _load(self, path: Path) -> None:
+        lines = path.read_text(encoding="utf-8").splitlines()
+        for number, line in enumerate(lines, start=1):
+            if not line.strip():
+                continue
+            try:
+                event = json.loads(line)
+            except ValueError:
+                if number == len(lines):
+                    break  # a line torn by a stop mid-write; nothing after it was acted on
+                raise ValueError(f"{path}:{number}: not a journal event") from None
+            self._events.append(event)
+            self.clock.advance_to(parse(event["at"]))
+
+    def append(self, event: dict[str, Any]) -> dict[str, Any]:
+        with self._lock:
+            entry = {**event, "at": stamp(self.clock.now())}
+            if self._path is not None:
+                with self._path.open("a", encoding="utf-8") as handle:
+                    handle.write(json.dumps(entry, sort_keys=True, separators=(",", ":")) + "\n")
+                    handle.flush()
+                    if self._fsync:
+                        os.fsync(handle.fileno())
+            self._events.append(entry)
+            return entry
+
+    def events(self, key: str | None = None) -> list[dict[str, Any]]:
+        with self._lock:
+            return [dict(item) for item in self._events if key is None or item.get("key") == key]
+
+    def keys(self) -> list[str]:
+        with self._lock:
+            return list(dict.fromkeys(item["key"] for item in self._events
+                                      if item.get("event") == "operation"))
+
+    def history(self, key: str) -> History:
+        history = History(key=key)
+        replies: dict[int, str] = {}
+        dispatched: list[dict[str, Any]] = []
+        for event in self.events(key):
+            kind = event.get("event")
+            if kind == "operation" and history.operation is None:
+                history.operation = event
+            elif kind == "dispatch":
+                dispatched.append(event)
+            elif kind == "reply":
+                replies[event["attempt"]] = event["reply"]
+            elif kind == "lookup":
+                history.lookups += 1
+                if event.get("answered"):
+                    history.reconciliations.append(
+                        {"at": event["at"], "final": event["final"], "idempotencyKey": key,
+                         "result": event["result"]})
+            elif kind == "outcome":
+                history.outcome = event["outcome"]
+        history.attempts = [Attempt(number=item["attempt"], key=key, args_digest=item["argsDigest"],
+                                    dispatched_at=item["at"], fencing_token=item.get("fencingToken"),
+                                    reply=replies.get(item["attempt"])) for item in dispatched]
+        return history
+
+    def trail(self, key: str) -> dict[str, Any]:
+        """The runtime's half of the record, without the owner's own effect count.
+
+        A dispatch with no recorded reply is exported as ``connection_lost``: the runtime has no reply.
+        An operation whose outcome was never reported is exported as ``unknown``.
+        """
+        history = self.history(key)
+        if history.operation is None:
+            raise KeyError(f"the journal holds no operation with key {key!r}")
+        operation = history.operation
+        attempts = []
+        for item in history.attempts:
+            attempt: dict[str, Any] = {"argsDigest": item.args_digest, "attempt": item.number,
+                                       "dispatchedAt": item.dispatched_at, "idempotencyKey": item.key,
+                                       "reply": item.reply or "connection_lost"}
+            if item.fencing_token is not None:
+                attempt["fencingToken"] = item.fencing_token
+            attempts.append(attempt)
+        events = self.events(key)
+        return {
+            "asOf": events[-1]["at"],
+            "attempts": attempts,
+            "contractId": CONTRACT_ID,
+            "effectOwner": operation["owner"],
+            "operation": {"action": operation["action"], "argsDigest": operation["argsDigest"],
+                          "id": key, "idempotencyKey": key},
+            "reconciliations": history.reconciliations,
+            "recordId": f"record-{key}",
+            "reported": {"outcome": history.outcome or "unknown"},
+        }
+
+    def record(self, key: str, *, owner_effects: int) -> dict[str, Any]:
+        """The whole uncertain-effect/v1 record: the trail joined with the owner's effect count."""
+        if isinstance(owner_effects, bool) or not isinstance(owner_effects, int) or owner_effects < 0:
+            raise ValueError("owner_effects is the owner's own count of effects, a whole number")
+        return {**self.trail(key), "ownerEffects": owner_effects}

probity_effects/langgraph.py

@@ -0,0 +1,115 @@
+"""A LangChain tool, for LangGraph's ToolNode or any agent built on it, guarded against lost replies.
+
+The tool never lets a lost reply escape as an exception, so neither a node retry policy nor a
+resumed thread nor a model that calls again can send the write blind: each call goes through the
+guard, which reads the journal first.
+"""
+
+import inspect
+import typing
+from collections.abc import Callable
+from typing import Any
+
+from probity_effects.guard import EffectGuard, EffectOutcomeUnknown, Outcome
+from probity_effects.journal import Journal
+from probity_effects.owner import EffectCall, EffectOwner
+
+ON_UNKNOWN = ("report", "raise")
+
+
+def effect_tool(owner: EffectOwner, *, journal: Journal | None = None, name: str | None = None,
+                description: str | None = None, key_from: str | None = None,
+                on_unknown: str = "report") -> Callable[[Callable[..., Any]], Any]:
+    """Decorate a function that performs one write into a guarded ``StructuredTool``.
+
+    The function's parameters, except one annotated ``EffectCall``, become the tool's arguments. That
+    parameter receives the dispatch: put ``effect.key`` in the request the owner deduplicates by. The key
+    is derived from the graph's ``thread_id``, the tool name and the arguments, unless ``key_from`` names an
+    argument that carries one. A worker holding a lease passes its token as ``fencing_token`` in the
+    graph's ``configurable``. An unknown outcome is reported to the model as a tool result, or raised
+    as :class:`EffectOutcomeUnknown` with ``on_unknown="raise"``.
+    """
+    if on_unknown not in ON_UNKNOWN:
+        raise ValueError(f"on_unknown is one of {ON_UNKNOWN}")
+    guard = EffectGuard(owner, journal)
+
+    def decorate(func: Callable[..., Any]) -> Any:
+        from langchain_core.runnables import RunnableConfig
+        from langchain_core.tools import StructuredTool
+        from langchain_core.tools.base import create_schema_from_function
+
+        tool_name = name or func.__name__
+        effect_param = _effect_parameter(func)
+        schema = create_schema_from_function(tool_name, func,
+                                             filter_args=[effect_param] if effect_param else [])
+        if key_from is not None and key_from not in schema.model_fields:
+            raise ValueError(f"key_from names {key_from!r}, which is not an argument of {tool_name}")
+
+        def body_args(call: EffectCall, args: dict[str, Any]) -> dict[str, Any]:
+            return {**args, effect_param: call} if effect_param else dict(args)
+
+        def settle(outcome: Outcome) -> Any:
+            return _render(tool_name, owner, outcome, on_unknown)
+
+        if inspect.iscoroutinefunction(func):
+            async def arun(config: RunnableConfig, **args: Any) -> Any:
+                outcome = await guard.aperform(tool_name, args, send=lambda call: func(**body_args(call, args)),
+                                               key=_key(args, key_from), scope=_scope(config),
+                                               fencing_token=_token(config))
+                return settle(outcome)
+
+            return StructuredTool.from_function(coroutine=arun, name=tool_name, args_schema=schema,
+                                                description=description or inspect.getdoc(func) or tool_name)
+
+        def run(config: RunnableConfig, **args: Any) -> Any:
+            outcome = guard.perform(tool_name, args, send=lambda call: func(**body_args(call, args)),
+                                    key=_key(args, key_from), scope=_scope(config), fencing_token=_token(config))
+            return settle(outcome)
+
+        return StructuredTool.from_function(func=run, name=tool_name, args_schema=schema,
+                                            description=description or inspect.getdoc(func) or tool_name)
+
+    return decorate
+
+
+def _effect_parameter(func: Callable[..., Any]) -> str | None:
+    try:
+        hints = typing.get_type_hints(func)
+    except (NameError, TypeError):
+        hints = {}
+    for parameter in inspect.signature(func).parameters.values():
+        annotation = hints.get(parameter.name, parameter.annotation)
+        if annotation is EffectCall or annotation == "EffectCall":
+            return parameter.name
+    return None
+
+
+def _configurable(config: Any) -> dict[str, Any]:
+    return dict((config or {}).get("configurable") or {})
+
+
+def _scope(config: Any) -> str:
+    thread = _configurable(config).get("thread_id")
+    return "" if thread is None else str(thread)
+
+
+def _token(config: Any) -> int | None:
+    token = _configurable(config).get("fencing_token")
+    return None if token is None else int(token)
+
+
+def _key(args: dict[str, Any], key_from: str | None) -> str | None:
+    return None if key_from is None else str(args[key_from])
+
+
+def _render(tool_name: str, owner: EffectOwner, outcome: Outcome, on_unknown: str) -> Any:
+    if outcome.outcome == "committed":
+        if outcome.sent and outcome.value is not None:
+            return outcome.value
+        return f"committed: {owner.id} holds this effect (idempotency key {outcome.key}); {outcome.reason}."
+    if outcome.outcome == "not_run":
+        return f"not run: {outcome.reason}."
+    if on_unknown == "raise":
+        raise EffectOutcomeUnknown(outcome)
+    return (f"outcome unknown: {outcome.reason}. The effect may have happened. Do not call {tool_name} again "
+            f"for this request; ask {owner.id} or the user what it holds (idempotency key {outcome.key}).")