langchain-signafaide 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_signafaide-0.1.0/.gitignore +11 -0
- langchain_signafaide-0.1.0/PKG-INFO +87 -0
- langchain_signafaide-0.1.0/README.md +67 -0
- langchain_signafaide-0.1.0/pyproject.toml +35 -0
- langchain_signafaide-0.1.0/src/langchain_signafaide/__init__.py +20 -0
- langchain_signafaide-0.1.0/src/langchain_signafaide/middleware.py +79 -0
- langchain_signafaide-0.1.0/src/langchain_signafaide/py.typed +0 -0
- langchain_signafaide-0.1.0/tests/test_middleware.py +123 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: langchain-signafaide
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: signafAIde for LangChain: signed, tamper-evident records of every tool call an agent makes.
|
|
5
|
+
Project-URL: Homepage, https://signafaide.co.uk
|
|
6
|
+
Project-URL: Documentation, https://signafaide.co.uk/docs/frameworks/langchain
|
|
7
|
+
Project-URL: Repository, https://github.com/AIME-TECH-UK-LIMITED/signafaide-sdk
|
|
8
|
+
Project-URL: Issues, https://github.com/AIME-TECH-UK-LIMITED/signafaide-sdk/issues
|
|
9
|
+
Author: AIME TECH UK LIMITED
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
Keywords: ai-agents,audit-trail,eu-ai-act,langchain,middleware,tamper-evident
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Security :: Cryptography
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: langchain<2,>=1.0
|
|
18
|
+
Requires-Dist: signafaide<0.4,>=0.3.1
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# langchain-signafaide
|
|
22
|
+
|
|
23
|
+
signafAIde for LangChain: a signed, tamper-evident record of every tool call an agent makes.
|
|
24
|
+
|
|
25
|
+
Each tool call writes a `decision` record before the tool runs and an `outcome` record after it. Records are signed with the agent's own key and chained on the machine that wrote them, in `.signafaide/`. Once a person opts in, a witness (the signafAIde service, or one you host) co-signs them and timestamps them publicly, so neither side can forge or edit a record alone. An auditor can verify the records without trusting either side. Records attest integrity and timing, never whether a decision was right.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install langchain-signafaide
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Needs Python 3.10 or later and LangChain 1.x.
|
|
34
|
+
|
|
35
|
+
## Middleware
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from langchain.agents import create_agent
|
|
39
|
+
from langchain_signafaide import SignafaideMiddleware
|
|
40
|
+
|
|
41
|
+
agent = create_agent(
|
|
42
|
+
model="openai:gpt-5",
|
|
43
|
+
tools=[get_weather, refund],
|
|
44
|
+
middleware=[SignafaideMiddleware(agent="support-bot", tools=[get_weather, refund])],
|
|
45
|
+
)
|
|
46
|
+
agent.invoke({"messages": [("user", "Refund order A1")]})
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Then check the records:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
signafaide verify
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
| Argument | What it does |
|
|
56
|
+
|---|---|
|
|
57
|
+
| `agent` | The name of the record chain. Use one name per running agent instance in production. Default: the project folder's name. |
|
|
58
|
+
| `tools` | Every tool the agent can call (tools or names). They are declared in the chain as recorded, so a verifier can report that all of the agent's tools are covered. Without it, a tool is declared when it is first called. |
|
|
59
|
+
| `strict` | A tool runs only once its decision has a receipt from the witness. If the witness cannot be reached, the call raises `NotRecorded` and the tool does not run. |
|
|
60
|
+
| `store_content` | Also send inputs and outputs to the witness, encrypted. Default: hashes only. |
|
|
61
|
+
|
|
62
|
+
The middleware works with `invoke` and `ainvoke`. A tool that raises is recorded with its error and the exception is raised unchanged.
|
|
63
|
+
|
|
64
|
+
## Tools outside an agent
|
|
65
|
+
|
|
66
|
+
For tools you call yourself, or pass to something other than `create_agent`:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from langchain_signafaide import wrap_tools
|
|
70
|
+
|
|
71
|
+
tools = wrap_tools([get_weather, refund], agent="support-bot")
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Use the middleware or `wrap_tools` for an agent's tools, not both, or each call is recorded twice.
|
|
75
|
+
|
|
76
|
+
## What is sent, and where
|
|
77
|
+
|
|
78
|
+
- Before a person opts in: nothing. Records are signed, chained and verified locally.
|
|
79
|
+
- After `signafaide init` (a free sandbox: no account, email or card) or with `SIGNAFAIDE_API_KEY` set: each record sends its content hash with its tool name, agent name and time to `https://api.signafaide.co.uk` (hosted in London), or to the address in `SIGNAFAIDE_URL`. Inputs and outputs are not sent unless stored content is turned on. Do not put personal data in tool or agent names.
|
|
80
|
+
- The middleware records the tool calls that go through the agent it is attached to. A call your application makes around the agent is not recorded.
|
|
81
|
+
|
|
82
|
+
## Links
|
|
83
|
+
|
|
84
|
+
- Documentation: https://signafaide.co.uk/docs/frameworks/langchain
|
|
85
|
+
- Privacy notice: https://signafaide.co.uk/legal/privacy
|
|
86
|
+
- Support: https://github.com/AIME-TECH-UK-LIMITED/signafaide-sdk/issues or sales@signafaide.co.uk
|
|
87
|
+
- Source: the `python-langchain/` folder of the signafAIde SDK repository. Apache-2.0.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# langchain-signafaide
|
|
2
|
+
|
|
3
|
+
signafAIde for LangChain: a signed, tamper-evident record of every tool call an agent makes.
|
|
4
|
+
|
|
5
|
+
Each tool call writes a `decision` record before the tool runs and an `outcome` record after it. Records are signed with the agent's own key and chained on the machine that wrote them, in `.signafaide/`. Once a person opts in, a witness (the signafAIde service, or one you host) co-signs them and timestamps them publicly, so neither side can forge or edit a record alone. An auditor can verify the records without trusting either side. Records attest integrity and timing, never whether a decision was right.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install langchain-signafaide
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Needs Python 3.10 or later and LangChain 1.x.
|
|
14
|
+
|
|
15
|
+
## Middleware
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from langchain.agents import create_agent
|
|
19
|
+
from langchain_signafaide import SignafaideMiddleware
|
|
20
|
+
|
|
21
|
+
agent = create_agent(
|
|
22
|
+
model="openai:gpt-5",
|
|
23
|
+
tools=[get_weather, refund],
|
|
24
|
+
middleware=[SignafaideMiddleware(agent="support-bot", tools=[get_weather, refund])],
|
|
25
|
+
)
|
|
26
|
+
agent.invoke({"messages": [("user", "Refund order A1")]})
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Then check the records:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
signafaide verify
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
| Argument | What it does |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `agent` | The name of the record chain. Use one name per running agent instance in production. Default: the project folder's name. |
|
|
38
|
+
| `tools` | Every tool the agent can call (tools or names). They are declared in the chain as recorded, so a verifier can report that all of the agent's tools are covered. Without it, a tool is declared when it is first called. |
|
|
39
|
+
| `strict` | A tool runs only once its decision has a receipt from the witness. If the witness cannot be reached, the call raises `NotRecorded` and the tool does not run. |
|
|
40
|
+
| `store_content` | Also send inputs and outputs to the witness, encrypted. Default: hashes only. |
|
|
41
|
+
|
|
42
|
+
The middleware works with `invoke` and `ainvoke`. A tool that raises is recorded with its error and the exception is raised unchanged.
|
|
43
|
+
|
|
44
|
+
## Tools outside an agent
|
|
45
|
+
|
|
46
|
+
For tools you call yourself, or pass to something other than `create_agent`:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from langchain_signafaide import wrap_tools
|
|
50
|
+
|
|
51
|
+
tools = wrap_tools([get_weather, refund], agent="support-bot")
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Use the middleware or `wrap_tools` for an agent's tools, not both, or each call is recorded twice.
|
|
55
|
+
|
|
56
|
+
## What is sent, and where
|
|
57
|
+
|
|
58
|
+
- Before a person opts in: nothing. Records are signed, chained and verified locally.
|
|
59
|
+
- After `signafaide init` (a free sandbox: no account, email or card) or with `SIGNAFAIDE_API_KEY` set: each record sends its content hash with its tool name, agent name and time to `https://api.signafaide.co.uk` (hosted in London), or to the address in `SIGNAFAIDE_URL`. Inputs and outputs are not sent unless stored content is turned on. Do not put personal data in tool or agent names.
|
|
60
|
+
- The middleware records the tool calls that go through the agent it is attached to. A call your application makes around the agent is not recorded.
|
|
61
|
+
|
|
62
|
+
## Links
|
|
63
|
+
|
|
64
|
+
- Documentation: https://signafaide.co.uk/docs/frameworks/langchain
|
|
65
|
+
- Privacy notice: https://signafaide.co.uk/legal/privacy
|
|
66
|
+
- Support: https://github.com/AIME-TECH-UK-LIMITED/signafaide-sdk/issues or sales@signafaide.co.uk
|
|
67
|
+
- Source: the `python-langchain/` folder of the signafAIde SDK repository. Apache-2.0.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.18"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "langchain-signafaide"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "signafAIde for LangChain: signed, tamper-evident records of every tool call an agent makes."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "Apache-2.0" }
|
|
12
|
+
authors = [{ name = "AIME TECH UK LIMITED" }]
|
|
13
|
+
dependencies = ["signafaide>=0.3.1,<0.4", "langchain>=1.0,<2"]
|
|
14
|
+
keywords = ["langchain", "middleware", "ai-agents", "audit-trail", "tamper-evident", "eu-ai-act"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"License :: OSI Approved :: Apache Software License",
|
|
17
|
+
"Topic :: Security :: Cryptography",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://signafaide.co.uk"
|
|
24
|
+
Documentation = "https://signafaide.co.uk/docs/frameworks/langchain"
|
|
25
|
+
Repository = "https://github.com/AIME-TECH-UK-LIMITED/signafaide-sdk"
|
|
26
|
+
Issues = "https://github.com/AIME-TECH-UK-LIMITED/signafaide-sdk/issues"
|
|
27
|
+
|
|
28
|
+
[dependency-groups]
|
|
29
|
+
dev = ["pytest>=7"]
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.wheel]
|
|
32
|
+
packages = ["src/langchain_signafaide"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""signafAIde for LangChain.
|
|
2
|
+
|
|
3
|
+
from langchain.agents import create_agent
|
|
4
|
+
from langchain_signafaide import SignafaideMiddleware
|
|
5
|
+
|
|
6
|
+
agent = create_agent(model, tools=tools, middleware=[SignafaideMiddleware(agent="support-bot")])
|
|
7
|
+
|
|
8
|
+
Every tool call the agent makes writes a signed ``decision`` record before the tool runs and an
|
|
9
|
+
``outcome`` record after it. Records are chained on the machine that wrote them, in
|
|
10
|
+
``.signafaide/``; nothing is sent until a person opts in (``signafaide init``, or
|
|
11
|
+
``SIGNAFAIDE_API_KEY``).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from signafaide import NotRecorded, wrap_tool, wrap_tools
|
|
15
|
+
|
|
16
|
+
from .middleware import SignafaideMiddleware
|
|
17
|
+
|
|
18
|
+
__version__ = "0.1.0"
|
|
19
|
+
|
|
20
|
+
__all__ = ["SignafaideMiddleware", "wrap_tools", "wrap_tool", "NotRecorded", "__version__"]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""``SignafaideMiddleware``: record every tool call a LangChain agent makes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Awaitable, Callable, Iterable, Optional
|
|
6
|
+
|
|
7
|
+
import signafaide
|
|
8
|
+
from langchain.agents.middleware import AgentMiddleware
|
|
9
|
+
from langchain_core.messages import ToolMessage
|
|
10
|
+
from signafaide import telemetry
|
|
11
|
+
|
|
12
|
+
# The recording step itself (decision, run, outcome) lives in the signafaide package. It has
|
|
13
|
+
# no public entry point for "record this call, which I run myself" yet, so this is the one
|
|
14
|
+
# place that reaches into it; tests/test_middleware.py fails if it changes shape.
|
|
15
|
+
from signafaide.decorator import _Spec
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _tool_output(result: Any) -> Any:
|
|
19
|
+
"""What the outcome records for the value a tool call produced."""
|
|
20
|
+
if isinstance(result, ToolMessage):
|
|
21
|
+
# A tool that failed where the agent handles tool errors comes back as an error
|
|
22
|
+
# message, not an exception: keep that visible in the record.
|
|
23
|
+
if getattr(result, "status", None) == "error":
|
|
24
|
+
return {"tool_error": result.content}
|
|
25
|
+
return result.content
|
|
26
|
+
update = getattr(result, "update", None) # a langgraph Command
|
|
27
|
+
return {"command": signafaide.to_jsonable(update)} if update is not None else signafaide.to_jsonable(result)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SignafaideMiddleware(AgentMiddleware):
|
|
31
|
+
"""Record every tool call of the agent as signed, chained signafAIde records.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
agent: The name of the record chain. Give one name per running agent instance in
|
|
35
|
+
production. Default: the project folder's name.
|
|
36
|
+
strict: A tool runs only once its decision has a receipt from the witness. When the
|
|
37
|
+
witness cannot be reached the call raises ``NotRecorded`` and the tool does not run.
|
|
38
|
+
store_content: Send inputs and outputs to the witness, encrypted, as well as their
|
|
39
|
+
hashes. Default: the project's setting (hashes only).
|
|
40
|
+
tools: The names of every tool the agent can call. They are declared in the chain as
|
|
41
|
+
recorded, so a verifier can say that all of the agent's tools are covered. Without
|
|
42
|
+
it, each tool is declared when it is first called.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
*,
|
|
48
|
+
agent: Optional[str] = None,
|
|
49
|
+
strict: bool = False,
|
|
50
|
+
store_content: Optional[bool] = None,
|
|
51
|
+
tools: Optional[Iterable[Any]] = None,
|
|
52
|
+
) -> None:
|
|
53
|
+
super().__init__()
|
|
54
|
+
self._agent = agent
|
|
55
|
+
self._strict = strict
|
|
56
|
+
self._store_content = store_content
|
|
57
|
+
self._specs: dict[str, _Spec] = {}
|
|
58
|
+
telemetry.ADAPTERS.add("langchain")
|
|
59
|
+
for t in tools or ():
|
|
60
|
+
self._spec(t if isinstance(t, str) else t.name)
|
|
61
|
+
|
|
62
|
+
def _spec(self, name: str) -> _Spec:
|
|
63
|
+
spec = self._specs.get(name)
|
|
64
|
+
if spec is None:
|
|
65
|
+
spec = self._specs[name] = _Spec(name, self._agent, self._strict, self._store_content, listed=True)
|
|
66
|
+
return spec
|
|
67
|
+
|
|
68
|
+
@staticmethod
|
|
69
|
+
def _call(request: Any) -> tuple[str, Any]:
|
|
70
|
+
call = request.tool_call
|
|
71
|
+
return call["name"], call.get("args", {})
|
|
72
|
+
|
|
73
|
+
def wrap_tool_call(self, request: Any, handler: Callable[[Any], Any]) -> Any:
|
|
74
|
+
name, args = self._call(request)
|
|
75
|
+
return self._spec(name).call(args, lambda: handler(request), _tool_output)
|
|
76
|
+
|
|
77
|
+
async def awrap_tool_call(self, request: Any, handler: Callable[[Any], Awaitable[Any]]) -> Any:
|
|
78
|
+
name, args = self._call(request)
|
|
79
|
+
return await self._spec(name).acall(args, lambda: handler(request), _tool_output)
|
|
File without changes
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import atexit
|
|
3
|
+
import inspect
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
from langchain.agents import create_agent
|
|
8
|
+
from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
|
|
9
|
+
from langchain_core.messages import AIMessage
|
|
10
|
+
from langchain_core.tools import tool
|
|
11
|
+
|
|
12
|
+
import signafaide
|
|
13
|
+
from langchain_signafaide import SignafaideMiddleware
|
|
14
|
+
from signafaide import decorator, runtime, telemetry
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class FakeModel(GenericFakeChatModel):
|
|
18
|
+
def bind_tools(self, tools, **kwargs):
|
|
19
|
+
return self
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def model_calling(*calls):
|
|
23
|
+
"""A model that asks for *calls* (name, args), one per turn, then answers."""
|
|
24
|
+
msgs = [AIMessage(content="", tool_calls=[{"name": n, "args": a, "id": f"call_{i}", "type": "tool_call"}]) for i, (n, a) in enumerate(calls)]
|
|
25
|
+
return FakeModel(messages=iter(msgs + [AIMessage(content="done")]))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@tool
|
|
29
|
+
def get_weather(city: str) -> str:
|
|
30
|
+
"""Weather for a city."""
|
|
31
|
+
if city == "Atlantis":
|
|
32
|
+
raise RuntimeError("no such city")
|
|
33
|
+
return f"sunny in {city}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@tool
|
|
37
|
+
def refund(order: str, amount: float) -> str:
|
|
38
|
+
"""Refund an order."""
|
|
39
|
+
return f"refunded {amount} on {order}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.fixture
|
|
43
|
+
def recorder(tmp_path, monkeypatch):
|
|
44
|
+
root = tmp_path / "Claims Project"
|
|
45
|
+
root.mkdir()
|
|
46
|
+
(root / ".git").mkdir()
|
|
47
|
+
monkeypatch.chdir(root)
|
|
48
|
+
for k in list(os.environ):
|
|
49
|
+
if k.startswith("SIGNAFAIDE_"):
|
|
50
|
+
monkeypatch.delenv(k)
|
|
51
|
+
monkeypatch.setenv("SIGNAFAIDE_TEST", "0") # exercise the real recorder, local only
|
|
52
|
+
monkeypatch.setattr(telemetry, "ADAPTERS", set())
|
|
53
|
+
decorator._reset_tool_registry()
|
|
54
|
+
r = runtime.configure(quiet=True, flush_interval=3600)
|
|
55
|
+
yield r
|
|
56
|
+
r._stopped = True
|
|
57
|
+
r._wake.set()
|
|
58
|
+
r._tool_stop.set()
|
|
59
|
+
atexit.unregister(r.close)
|
|
60
|
+
runtime._default = None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def records(rec, agent):
|
|
64
|
+
(chain,) = [c for c in rec.state.chains() if c.rsplit("/", 1)[1] == agent]
|
|
65
|
+
return list(rec.state.read_all(chain))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def calls(recs):
|
|
69
|
+
return [(r["envelope"]["kind"], r["envelope"]["name"]) for r in recs if r["envelope"]["kind"] != "note"]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_agent_tool_calls_are_recorded(recorder):
|
|
73
|
+
agent = create_agent(model_calling(("get_weather", {"city": "Paris"}), ("refund", {"order": "A1", "amount": 12.5})), tools=[get_weather, refund], middleware=[SignafaideMiddleware(agent="support-bot")])
|
|
74
|
+
out = agent.invoke({"messages": [("user", "weather, then refund")]})
|
|
75
|
+
assert out["messages"][-1].content == "done"
|
|
76
|
+
recs = records(recorder, "support-bot")
|
|
77
|
+
assert calls(recs) == [("decision", "get_weather"), ("outcome", "get_weather"), ("decision", "refund"), ("outcome", "refund")]
|
|
78
|
+
body = [r for r in recs if r["envelope"]["kind"] != "note"]
|
|
79
|
+
assert body[0]["content"]["input"] == {"city": "Paris"} and body[1]["content"]["output"] == "sunny in Paris"
|
|
80
|
+
assert body[2]["content"]["input"] == {"order": "A1", "amount": 12.5} and body[3]["content"]["output"] == "refunded 12.5 on A1"
|
|
81
|
+
assert signafaide.verify().ok
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_async_agent(recorder):
|
|
85
|
+
agent = create_agent(model_calling(("get_weather", {"city": "Oslo"})), tools=[get_weather], middleware=[SignafaideMiddleware(agent="support-bot")])
|
|
86
|
+
asyncio.run(agent.ainvoke({"messages": [("user", "weather")]}))
|
|
87
|
+
body = [r for r in records(recorder, "support-bot") if r["envelope"]["kind"] != "note"]
|
|
88
|
+
assert calls(body) == [("decision", "get_weather"), ("outcome", "get_weather")]
|
|
89
|
+
assert body[1]["content"]["output"] == "sunny in Oslo"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_tool_error_is_recorded_and_raised(recorder):
|
|
93
|
+
agent = create_agent(model_calling(("get_weather", {"city": "Atlantis"})), tools=[get_weather], middleware=[SignafaideMiddleware(agent="support-bot")])
|
|
94
|
+
with pytest.raises(RuntimeError, match="no such city"):
|
|
95
|
+
agent.invoke({"messages": [("user", "weather")]})
|
|
96
|
+
body = [r for r in records(recorder, "support-bot") if r["envelope"]["kind"] != "note"]
|
|
97
|
+
assert body[1]["content"]["error"] == {"type": "RuntimeError", "message": "no such city"}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_declared_tools_cover_the_agent(recorder):
|
|
101
|
+
mw = SignafaideMiddleware(agent="support-bot", tools=[get_weather, refund])
|
|
102
|
+
agent = create_agent(model_calling(("get_weather", {"city": "Rome"})), tools=[get_weather, refund], middleware=[mw])
|
|
103
|
+
agent.invoke({"messages": [("user", "weather")]})
|
|
104
|
+
notes = [r["envelope"]["name"] for r in records(recorder, "support-bot") if r["envelope"]["kind"] == "note"]
|
|
105
|
+
# refund was never called, and is still declared as one of the agent's recorded tools.
|
|
106
|
+
assert "signafaide.tool/refund" in notes and "signafaide.tool/get_weather" in notes
|
|
107
|
+
assert not [n for n in notes if n.startswith("signafaide.tool_unrecorded/")]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_recording_seam_has_the_shape_this_package_uses():
|
|
111
|
+
spec = inspect.signature(decorator._Spec)
|
|
112
|
+
assert list(spec.parameters)[:4] == ["name", "agent", "strict", "store_content"] and "listed" in spec.parameters
|
|
113
|
+
for fn in (decorator._Spec.call, decorator._Spec.acall):
|
|
114
|
+
assert list(inspect.signature(fn).parameters) == ["self", "args", "fn", "output"]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_handled_tool_error_stays_visible_in_the_record():
|
|
118
|
+
from langchain_core.messages import ToolMessage
|
|
119
|
+
|
|
120
|
+
from langchain_signafaide.middleware import _tool_output
|
|
121
|
+
|
|
122
|
+
assert _tool_output(ToolMessage(content="ok", tool_call_id="c1")) == "ok"
|
|
123
|
+
assert _tool_output(ToolMessage(content="boom", tool_call_id="c1", status="error")) == {"tool_error": "boom"}
|