toolreceipt 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/PKG-INFO +47 -16
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/README.md +46 -15
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/pyproject.toml +1 -1
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/__init__.py +3 -2
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/claims.py +179 -11
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/langgraph.py +4 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/lexicon.json +2 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/llm.py +40 -9
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/receipts.py +54 -2
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/.gitignore +0 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/LICENSE +0 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/scripts/eval_llm.py +0 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/diff.py +0 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/ledger.py +0 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/py.typed +0 -0
- {toolreceipt-0.2.0 → toolreceipt-0.3.0}/toolreceipt/receipt.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: toolreceipt
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Catch LLM agents claiming actions they never took. Ledger + claim extraction + receipt for LangGraph.
|
|
5
5
|
Project-URL: Homepage, https://github.com/yohannesjoya/toolreceipt
|
|
6
6
|
Project-URL: Repository, https://github.com/yohannesjoya/toolreceipt
|
|
@@ -62,11 +62,21 @@ advice ("you can create a ticket"), conditionals and offers are not claims.
|
|
|
62
62
|
- **Non-English replies.** The default extractor is English only. `LLMExtractor` reads
|
|
63
63
|
other languages, but its guards are English only (see below).
|
|
64
64
|
- **Streaming and mid-run checks.** Only the final reply is checked.
|
|
65
|
-
- **Tool names without a verb
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
65
|
+
- **Tool names without a verb, when the description does not lead with one.** A tool
|
|
66
|
+
called `jira` or `email_customer` is matched through the first verb of its description
|
|
67
|
+
("Create a ticket in Jira", "Email the customer"). Only the first word counts, and a
|
|
68
|
+
read verb anywhere in the first clause, up to the first `.`, `;` or "and" ("Look up an
|
|
69
|
+
order and email the customer"), rules the tool out. Only the verb's direct object
|
|
70
|
+
("a ticket") is evidence that a claim is about this tool; the rest of the description
|
|
71
|
+
("for the customer") is not. A description that opens with something else ("Helps the customer
|
|
72
|
+
by creating a ticket", "Email address validation") or is in another language leaves the
|
|
73
|
+
tool unmatched, and claims about it land in `unmatched` (informational, they never
|
|
74
|
+
affect the verdict). Put the verb first, or use `LLMExtractor`, which matches on the
|
|
75
|
+
whole description. On the post-hoc path, pass `known_tools` with descriptions: a ledger
|
|
76
|
+
alone carries none.
|
|
77
|
+
- **Loose tool matching.** A claim maps to a tool only when the tool's name, or the first
|
|
78
|
+
verb of its description, shares the claim's action verb family (`send`, `create`,
|
|
79
|
+
`update`, `delete`, `refund`, ...).
|
|
70
80
|
- **Loose paraphrases of an action done with a different tool.** "I've raised this with our
|
|
71
81
|
team" after `escalate_ticket` fits the verb family of `create_ticket` but names nothing in
|
|
72
82
|
it. When a claim matches a tool on the verb family alone and that tool never ran, the claim
|
|
@@ -104,9 +114,10 @@ Before tuning, a milder pressure prompt produced no false claim in 10 of 10 runs
|
|
|
104
114
|
prompt was chosen on the first 5 of the 10 pooled runs, so those 5 are not held out.
|
|
105
115
|
|
|
106
116
|
In 10 of 10 live honest runs on 2026-10-19 every tool succeeded and the receipt was `ok`.
|
|
107
|
-
On the held-out set (`fixtures/heldout/`: those 10 honest replies
|
|
108
|
-
adversarial negatives, labelled before any rule
|
|
109
|
-
with no claim on any adversarial negative
|
|
117
|
+
On the held-out set (`fixtures/heldout/`: those 10 honest replies, three more written for
|
|
118
|
+
0.3's verbless tools, and 12 hand-written adversarial negatives, labelled before any rule
|
|
119
|
+
change) precision is 1.00 (23 of 23 claims) with no claim on any adversarial negative and no
|
|
120
|
+
honest reply flagged. The 10 live honest replies are near-identical in
|
|
110
121
|
phrasing ("I've issued a $64 refund ... and sent a confirmation email"), so this precision
|
|
111
122
|
figure is weak evidence. `fixtures/check/` adds hand-written honest replies with more varied
|
|
112
123
|
phrasing. The recorded runs in `demo/replays/` back the demo's Replay button.
|
|
@@ -122,14 +133,16 @@ temperature 0, recorded 2026-10-22 and replayed offline by `tests/test_probe_rep
|
|
|
122
133
|
|
|
123
134
|
| Set | Extractor | Claims found | Precision | Fabrications caught | False alarms on traps |
|
|
124
135
|
|---|---|---|---|---|---|
|
|
125
|
-
| dev (114) | rules only |
|
|
136
|
+
| dev (114) | rules only | 36 / 113 (32%) | 0.95 | 15 / 67 (22%) | 2 / 23 |
|
|
126
137
|
| dev (114) | rules + LLM, default (`suspected`) | 101 / 113 (89%) | 0.97 | 60 / 67 (90%) | 2 / 23 (0 added by the LLM) |
|
|
127
138
|
| dev (114) | rules + LLM, `llm_blocks=True` | 101 / 113 (89%) | 0.97 | 60 / 67 (90%) | 2 / 23 (0 added by the LLM) |
|
|
128
|
-
| held-out (30) | rules only |
|
|
139
|
+
| held-out (30) | rules only | 5 / 35 (14%) | 1.00 | 1 / 18 (6%) | 0 / 7 |
|
|
129
140
|
| held-out (30) | rules + LLM, default (`suspected`) | 31 / 35 (89%) | 0.97 | 15 / 18 (83%) | 1 / 7 |
|
|
130
141
|
| held-out (30) | rules + LLM, `llm_blocks=True` | 31 / 35 (89%) | 0.97 | 15 / 18 (83%) | 1 / 7 |
|
|
131
142
|
|
|
132
|
-
|
|
143
|
+
Rules-only rows re-measured for 0.3.0, when verbless probe tools (`tool_b`, `tool_c`) became
|
|
144
|
+
matchable through their descriptions; the LLM rows are unchanged. "Caught" means the receipt
|
|
145
|
+
flags a labelled fabrication: `verdict == "fabricated"`, or a
|
|
133
146
|
non-empty `suspected` under the default. The two dev trap alarms are the rules' own (a quoted
|
|
134
147
|
draft and "refunded in March"), unchanged from 0.1. The held-out alarm is the model reading
|
|
135
148
|
"the refund bounced" as a refund; the misses are vague replies ("Both of those are sorted
|
|
@@ -168,6 +181,22 @@ if not receipt.ok:
|
|
|
168
181
|
`Receipts` per run. Your tools and graph stay unchanged, apart from
|
|
169
182
|
`ToolNode(tools, handle_tool_errors=True)`.
|
|
170
183
|
|
|
184
|
+
Async graphs work the same way:
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
rc = Receipts() # one per run: config() binds the ledger to this instance
|
|
188
|
+
result = await graph.ainvoke(inputs, config=rc.config())
|
|
189
|
+
receipt = await rc.acheck(final_reply(result))
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
`acheck` returns the same receipt as `check`. With the rules alone it does no I/O; with an
|
|
193
|
+
`LLMExtractor` the model call runs on a thread so the event loop stays free. `astream` is
|
|
194
|
+
recorded the same way, and a cancelled run closes its open tool call as an error.
|
|
195
|
+
|
|
196
|
+
On Python 3.10, under `ainvoke` or `astream`, LangGraph does not pass the run config into
|
|
197
|
+
nodes implicitly. Call your model with the node's `config` argument there, or the tools it
|
|
198
|
+
binds are not recorded as known.
|
|
199
|
+
|
|
171
200
|
## With an LLM extractor
|
|
172
201
|
|
|
173
202
|
The default extractor is rules only: no key, no network, deterministic. It reads direct
|
|
@@ -248,8 +277,9 @@ no result is recorded as an error. Timestamps are `None` on this path.
|
|
|
248
277
|
|
|
249
278
|
Pass `known_tools`. A ledger only knows the tools that ran, so without `known_tools` a claim
|
|
250
279
|
about a tool that was never called ("I emailed you" when `send_email` never ran) has nothing
|
|
251
|
-
to match and cannot be flagged. Names are enough; `(name, description)`
|
|
252
|
-
OpenAI-style tool dicts also work
|
|
280
|
+
to match and cannot be flagged. Names are enough for verb-named tools; `(name, description)`
|
|
281
|
+
pairs and OpenAI-style tool dicts also work, and a tool whose name has no verb (`jira`) is
|
|
282
|
+
matched only when its description is given. In a live run (`Receipts.config()`), bound tools are
|
|
253
283
|
discovered automatically.
|
|
254
284
|
|
|
255
285
|
Without LangGraph, build a `Ledger` of `LedgerEntry` objects yourself and call `check()`.
|
|
@@ -301,12 +331,13 @@ kept.
|
|
|
301
331
|
|
|
302
332
|
## Roadmap
|
|
303
333
|
|
|
304
|
-
Done in 0.2: LLM-backed extractor behind the existing `ClaimExtractor` protocol.
|
|
334
|
+
Done in 0.2: LLM-backed extractor behind the existing `ClaimExtractor` protocol. Done in
|
|
335
|
+
0.3: async API; verbless tool names matched through their descriptions.
|
|
305
336
|
|
|
306
337
|
- TypeScript port, sharing the JSON fixtures as the contract
|
|
307
338
|
- Adapters: Mastra, CrewAI, Vercel AI SDK
|
|
308
339
|
- Argument check ("emailed Bob" against `to="alice"`)
|
|
309
|
-
-
|
|
340
|
+
- Framework-free message input (OpenAI-style message lists)
|
|
310
341
|
|
|
311
342
|
## Contributing
|
|
312
343
|
|
|
@@ -41,11 +41,21 @@ advice ("you can create a ticket"), conditionals and offers are not claims.
|
|
|
41
41
|
- **Non-English replies.** The default extractor is English only. `LLMExtractor` reads
|
|
42
42
|
other languages, but its guards are English only (see below).
|
|
43
43
|
- **Streaming and mid-run checks.** Only the final reply is checked.
|
|
44
|
-
- **Tool names without a verb
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
- **Tool names without a verb, when the description does not lead with one.** A tool
|
|
45
|
+
called `jira` or `email_customer` is matched through the first verb of its description
|
|
46
|
+
("Create a ticket in Jira", "Email the customer"). Only the first word counts, and a
|
|
47
|
+
read verb anywhere in the first clause, up to the first `.`, `;` or "and" ("Look up an
|
|
48
|
+
order and email the customer"), rules the tool out. Only the verb's direct object
|
|
49
|
+
("a ticket") is evidence that a claim is about this tool; the rest of the description
|
|
50
|
+
("for the customer") is not. A description that opens with something else ("Helps the customer
|
|
51
|
+
by creating a ticket", "Email address validation") or is in another language leaves the
|
|
52
|
+
tool unmatched, and claims about it land in `unmatched` (informational, they never
|
|
53
|
+
affect the verdict). Put the verb first, or use `LLMExtractor`, which matches on the
|
|
54
|
+
whole description. On the post-hoc path, pass `known_tools` with descriptions: a ledger
|
|
55
|
+
alone carries none.
|
|
56
|
+
- **Loose tool matching.** A claim maps to a tool only when the tool's name, or the first
|
|
57
|
+
verb of its description, shares the claim's action verb family (`send`, `create`,
|
|
58
|
+
`update`, `delete`, `refund`, ...).
|
|
49
59
|
- **Loose paraphrases of an action done with a different tool.** "I've raised this with our
|
|
50
60
|
team" after `escalate_ticket` fits the verb family of `create_ticket` but names nothing in
|
|
51
61
|
it. When a claim matches a tool on the verb family alone and that tool never ran, the claim
|
|
@@ -83,9 +93,10 @@ Before tuning, a milder pressure prompt produced no false claim in 10 of 10 runs
|
|
|
83
93
|
prompt was chosen on the first 5 of the 10 pooled runs, so those 5 are not held out.
|
|
84
94
|
|
|
85
95
|
In 10 of 10 live honest runs on 2026-10-19 every tool succeeded and the receipt was `ok`.
|
|
86
|
-
On the held-out set (`fixtures/heldout/`: those 10 honest replies
|
|
87
|
-
adversarial negatives, labelled before any rule
|
|
88
|
-
with no claim on any adversarial negative
|
|
96
|
+
On the held-out set (`fixtures/heldout/`: those 10 honest replies, three more written for
|
|
97
|
+
0.3's verbless tools, and 12 hand-written adversarial negatives, labelled before any rule
|
|
98
|
+
change) precision is 1.00 (23 of 23 claims) with no claim on any adversarial negative and no
|
|
99
|
+
honest reply flagged. The 10 live honest replies are near-identical in
|
|
89
100
|
phrasing ("I've issued a $64 refund ... and sent a confirmation email"), so this precision
|
|
90
101
|
figure is weak evidence. `fixtures/check/` adds hand-written honest replies with more varied
|
|
91
102
|
phrasing. The recorded runs in `demo/replays/` back the demo's Replay button.
|
|
@@ -101,14 +112,16 @@ temperature 0, recorded 2026-10-22 and replayed offline by `tests/test_probe_rep
|
|
|
101
112
|
|
|
102
113
|
| Set | Extractor | Claims found | Precision | Fabrications caught | False alarms on traps |
|
|
103
114
|
|---|---|---|---|---|---|
|
|
104
|
-
| dev (114) | rules only |
|
|
115
|
+
| dev (114) | rules only | 36 / 113 (32%) | 0.95 | 15 / 67 (22%) | 2 / 23 |
|
|
105
116
|
| dev (114) | rules + LLM, default (`suspected`) | 101 / 113 (89%) | 0.97 | 60 / 67 (90%) | 2 / 23 (0 added by the LLM) |
|
|
106
117
|
| dev (114) | rules + LLM, `llm_blocks=True` | 101 / 113 (89%) | 0.97 | 60 / 67 (90%) | 2 / 23 (0 added by the LLM) |
|
|
107
|
-
| held-out (30) | rules only |
|
|
118
|
+
| held-out (30) | rules only | 5 / 35 (14%) | 1.00 | 1 / 18 (6%) | 0 / 7 |
|
|
108
119
|
| held-out (30) | rules + LLM, default (`suspected`) | 31 / 35 (89%) | 0.97 | 15 / 18 (83%) | 1 / 7 |
|
|
109
120
|
| held-out (30) | rules + LLM, `llm_blocks=True` | 31 / 35 (89%) | 0.97 | 15 / 18 (83%) | 1 / 7 |
|
|
110
121
|
|
|
111
|
-
|
|
122
|
+
Rules-only rows re-measured for 0.3.0, when verbless probe tools (`tool_b`, `tool_c`) became
|
|
123
|
+
matchable through their descriptions; the LLM rows are unchanged. "Caught" means the receipt
|
|
124
|
+
flags a labelled fabrication: `verdict == "fabricated"`, or a
|
|
112
125
|
non-empty `suspected` under the default. The two dev trap alarms are the rules' own (a quoted
|
|
113
126
|
draft and "refunded in March"), unchanged from 0.1. The held-out alarm is the model reading
|
|
114
127
|
"the refund bounced" as a refund; the misses are vague replies ("Both of those are sorted
|
|
@@ -147,6 +160,22 @@ if not receipt.ok:
|
|
|
147
160
|
`Receipts` per run. Your tools and graph stay unchanged, apart from
|
|
148
161
|
`ToolNode(tools, handle_tool_errors=True)`.
|
|
149
162
|
|
|
163
|
+
Async graphs work the same way:
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
rc = Receipts() # one per run: config() binds the ledger to this instance
|
|
167
|
+
result = await graph.ainvoke(inputs, config=rc.config())
|
|
168
|
+
receipt = await rc.acheck(final_reply(result))
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`acheck` returns the same receipt as `check`. With the rules alone it does no I/O; with an
|
|
172
|
+
`LLMExtractor` the model call runs on a thread so the event loop stays free. `astream` is
|
|
173
|
+
recorded the same way, and a cancelled run closes its open tool call as an error.
|
|
174
|
+
|
|
175
|
+
On Python 3.10, under `ainvoke` or `astream`, LangGraph does not pass the run config into
|
|
176
|
+
nodes implicitly. Call your model with the node's `config` argument there, or the tools it
|
|
177
|
+
binds are not recorded as known.
|
|
178
|
+
|
|
150
179
|
## With an LLM extractor
|
|
151
180
|
|
|
152
181
|
The default extractor is rules only: no key, no network, deterministic. It reads direct
|
|
@@ -227,8 +256,9 @@ no result is recorded as an error. Timestamps are `None` on this path.
|
|
|
227
256
|
|
|
228
257
|
Pass `known_tools`. A ledger only knows the tools that ran, so without `known_tools` a claim
|
|
229
258
|
about a tool that was never called ("I emailed you" when `send_email` never ran) has nothing
|
|
230
|
-
to match and cannot be flagged. Names are enough; `(name, description)`
|
|
231
|
-
OpenAI-style tool dicts also work
|
|
259
|
+
to match and cannot be flagged. Names are enough for verb-named tools; `(name, description)`
|
|
260
|
+
pairs and OpenAI-style tool dicts also work, and a tool whose name has no verb (`jira`) is
|
|
261
|
+
matched only when its description is given. In a live run (`Receipts.config()`), bound tools are
|
|
232
262
|
discovered automatically.
|
|
233
263
|
|
|
234
264
|
Without LangGraph, build a `Ledger` of `LedgerEntry` objects yourself and call `check()`.
|
|
@@ -280,12 +310,13 @@ kept.
|
|
|
280
310
|
|
|
281
311
|
## Roadmap
|
|
282
312
|
|
|
283
|
-
Done in 0.2: LLM-backed extractor behind the existing `ClaimExtractor` protocol.
|
|
313
|
+
Done in 0.2: LLM-backed extractor behind the existing `ClaimExtractor` protocol. Done in
|
|
314
|
+
0.3: async API; verbless tool names matched through their descriptions.
|
|
284
315
|
|
|
285
316
|
- TypeScript port, sharing the JSON fixtures as the contract
|
|
286
317
|
- Adapters: Mastra, CrewAI, Vercel AI SDK
|
|
287
318
|
- Argument check ("emailed Bob" against `to="alice"`)
|
|
288
|
-
-
|
|
319
|
+
- Framework-free message input (OpenAI-style message lists)
|
|
289
320
|
|
|
290
321
|
## Contributing
|
|
291
322
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "toolreceipt"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Catch LLM agents claiming actions they never took. Ledger + claim extraction + receipt for LangGraph."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -4,9 +4,9 @@ from toolreceipt.claims import Claim, ClaimExtractor, RuleExtractor, ToolSpec
|
|
|
4
4
|
from toolreceipt.ledger import Ledger, LedgerEntry
|
|
5
5
|
from toolreceipt.llm import LLMExtractor
|
|
6
6
|
from toolreceipt.receipt import Receipt, Verdict
|
|
7
|
-
from toolreceipt.receipts import Receipts, check
|
|
7
|
+
from toolreceipt.receipts import Receipts, acheck, check
|
|
8
8
|
|
|
9
|
-
__version__ = "0.
|
|
9
|
+
__version__ = "0.3.0"
|
|
10
10
|
|
|
11
11
|
__all__ = [
|
|
12
12
|
"Claim",
|
|
@@ -20,5 +20,6 @@ __all__ = [
|
|
|
20
20
|
"ToolSpec",
|
|
21
21
|
"Verdict",
|
|
22
22
|
"__version__",
|
|
23
|
+
"acheck",
|
|
23
24
|
"check",
|
|
24
25
|
]
|
|
@@ -470,6 +470,115 @@ def _words(text: str) -> list[str]:
|
|
|
470
470
|
return [_stem(w) for w in _WORD.findall(_normalize(text).lower())]
|
|
471
471
|
|
|
472
472
|
|
|
473
|
+
def _stem_verb(word: str) -> str:
|
|
474
|
+
"""``_stem`` plus third-person handling: "searches" -> "search", "queries" -> "query",
|
|
475
|
+
"closes" -> "close", "processes" -> "process"."""
|
|
476
|
+
w = word.lower()
|
|
477
|
+
if len(w) > 4 and w.endswith("ies"):
|
|
478
|
+
return w[:-3] + "y"
|
|
479
|
+
if len(w) > 4 and w.endswith("es") and w[:-2].endswith(("sh", "ch", "x", "z", "ss", "o")):
|
|
480
|
+
return w[:-2]
|
|
481
|
+
return _stem(w)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
_LEAD_PREFIX = re.compile(r"^\s*[A-Za-z0-9_-]+:\s*")
|
|
485
|
+
_LEAD_NOUN_FOLLOWERS = frozenset({
|
|
486
|
+
"the", "a", "an", "your", "our", "my", "his", "her", "their", "its", "this", "that",
|
|
487
|
+
"these", "those", "me", "you", "us", "them", "it", "all", "each", "every", "any", "one",
|
|
488
|
+
})
|
|
489
|
+
_CLAUSE_END = re.compile(r"[.;]| and ")
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _description_clause(description: str) -> str:
|
|
493
|
+
"""Text up to the first ``.``, ``;`` or `` and ``."""
|
|
494
|
+
return _CLAUSE_END.split(_normalize(description), maxsplit=1)[0]
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _lead_words(description: str, skip: Sequence[str]) -> list[str]:
|
|
498
|
+
"""Lowercase words of the first clause after an optional single-word ``<Name>:`` prefix
|
|
499
|
+
and one skip phrase (longest first)."""
|
|
500
|
+
text = _LEAD_PREFIX.sub("", _description_clause(description), count=1).lower().strip()
|
|
501
|
+
for phrase in sorted(skip, key=len, reverse=True):
|
|
502
|
+
if text == phrase or text.startswith(phrase + " "):
|
|
503
|
+
text = text[len(phrase):].strip()
|
|
504
|
+
break
|
|
505
|
+
return _WORD.findall(text)
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def description_family(description: str, lexicon: Mapping[str, Any] | None = None) -> str | None:
|
|
509
|
+
"""The family named by a description's lead verb (PRD 03 §7.2 rule 2), or ``None``.
|
|
510
|
+
|
|
511
|
+
The lead verb is the first content word of the first clause, after an optional
|
|
512
|
+
``<Name>:`` prefix and one ``description_lead_skip`` phrase. It names a family when its
|
|
513
|
+
stem is one of the family's verb tokens or reply forms (a two-word form such as "set
|
|
514
|
+
up" is tested first), or a noun token listed in ``description_lead_nouns``. A read verb
|
|
515
|
+
as the lead word, or anywhere in the first clause, gives ``None``.
|
|
516
|
+
"""
|
|
517
|
+
lex = _default_lexicon() if lexicon is None else lexicon
|
|
518
|
+
words = _lead_words(description, lex.get("description_lead_skip", []))
|
|
519
|
+
if not words:
|
|
520
|
+
return None
|
|
521
|
+
reads = {_stem(v) for v in lex.get("read_verbs", ())}
|
|
522
|
+
if any(_stem_verb(w) in reads for w in words):
|
|
523
|
+
return None
|
|
524
|
+
families: Mapping[str, Any] = lex["families"]
|
|
525
|
+
two = " ".join(_stem_verb(w) for w in words[:2])
|
|
526
|
+
for name, fam in families.items():
|
|
527
|
+
forms = fam.get("forms", ())
|
|
528
|
+
if any(" " in f and " ".join(_stem_verb(w) for w in f.split()) == two for f in forms):
|
|
529
|
+
return name
|
|
530
|
+
stem = _stem_verb(words[0])
|
|
531
|
+
for name, fam in families.items():
|
|
532
|
+
single = {_stem(f) for f in fam.get("forms", ()) if " " not in f}
|
|
533
|
+
if stem in _verb_tokens(name, lex) or stem in single:
|
|
534
|
+
return name
|
|
535
|
+
# A noun token is an imperative verb only when an object follows it ("Email the
|
|
536
|
+
# customer"), never as a modifier ("Email address validation", "Issue tracker").
|
|
537
|
+
follower = words[1] if len(words) > 1 else None
|
|
538
|
+
if stem in {_stem(n) for n in lex.get("description_lead_nouns", ())} and (
|
|
539
|
+
follower in _LEAD_NOUN_FOLLOWERS
|
|
540
|
+
):
|
|
541
|
+
return _noun_families(lex).get(stem)
|
|
542
|
+
return None
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
# Words that end a description's direct object: prepositions and clause openers.
|
|
546
|
+
_OBJECT_END = frozenset({
|
|
547
|
+
"to", "for", "of", "in", "on", "at", "by", "with", "from", "into", "about", "through",
|
|
548
|
+
"via", "as", "using", "when", "if", "that", "which", "who", "where", "so", "or",
|
|
549
|
+
})
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _description_object(
|
|
553
|
+
description: str, lexicon: Mapping[str, Any] | None = None
|
|
554
|
+
) -> list[str]:
|
|
555
|
+
"""Stemmed words of the lead verb's direct object (PRD 03 §7.2 rule 3), or ``[]``.
|
|
556
|
+
|
|
557
|
+
The words after the lead verb (both words of a two-word form such as "set up") and any
|
|
558
|
+
determiners, up to the first preposition or the clause end: "Create a ticket in Jira"
|
|
559
|
+
gives ``["ticket"]``, "Send an email to the customer about their order" gives
|
|
560
|
+
``["email"]``. Empty when the description names no family.
|
|
561
|
+
"""
|
|
562
|
+
lex = _default_lexicon() if lexicon is None else lexicon
|
|
563
|
+
if description_family(description, lex) is None:
|
|
564
|
+
return []
|
|
565
|
+
words = _lead_words(description, lex.get("description_lead_skip", []))
|
|
566
|
+
two = " ".join(_stem_verb(w) for w in words[:2])
|
|
567
|
+
two_word_forms = {
|
|
568
|
+
" ".join(_stem_verb(w) for w in f.split())
|
|
569
|
+
for fam in lex["families"].values()
|
|
570
|
+
for f in fam.get("forms", ())
|
|
571
|
+
if " " in f
|
|
572
|
+
}
|
|
573
|
+
obj: list[str] = []
|
|
574
|
+
for w in words[2:] if two in two_word_forms else words[1:]:
|
|
575
|
+
if w in _OBJECT_END:
|
|
576
|
+
break
|
|
577
|
+
if w not in _LEAD_NOUN_FOLLOWERS:
|
|
578
|
+
obj.append(_stem(w))
|
|
579
|
+
return obj
|
|
580
|
+
|
|
581
|
+
|
|
473
582
|
def tokenize_tool_name(name: str) -> list[str]:
|
|
474
583
|
"""Split a tool name on ``_``, ``-``, other separators and camelCase; lowercase, singular."""
|
|
475
584
|
return [_stem(part) for chunk in _NAME_SPLIT.split(name) for part in _NAME_PART.findall(chunk)]
|
|
@@ -519,10 +628,32 @@ def family_ran(family: str, tools: Iterable[str], lexicon: Mapping[str, Any] | N
|
|
|
519
628
|
return any(verbs & set(tokenize_tool_name(t)) for t in tools)
|
|
520
629
|
|
|
521
630
|
|
|
631
|
+
def _is_verb_named(name_tokens: Sequence[str], lexicon: Mapping[str, Any]) -> bool:
|
|
632
|
+
"""True when a tool name carries a verb token of any family or a read verb."""
|
|
633
|
+
tokens = set(name_tokens)
|
|
634
|
+
if tokens & {_stem(v) for v in lexicon.get("read_verbs", ())}:
|
|
635
|
+
return True
|
|
636
|
+
return any(tokens & _verb_tokens(f, lexicon) for f in lexicon["families"])
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _tool_verb_tokens(spec: ToolSpec, lexicon: Mapping[str, Any]) -> set[str]:
|
|
640
|
+
"""The tokens a tool is matched on (PRD 03 §7.2 rule 1).
|
|
641
|
+
|
|
642
|
+
A verb-named tool (``_is_verb_named``) is matched on its name tokens, exactly as before
|
|
643
|
+
0.3. A verbless name (``jira``, ``email_customer``) is matched on the verb tokens of the
|
|
644
|
+
family its description's lead verb names (``description_family``), or on nothing.
|
|
645
|
+
"""
|
|
646
|
+
name = tokenize_tool_name(spec.name)
|
|
647
|
+
if _is_verb_named(name, lexicon):
|
|
648
|
+
return set(name)
|
|
649
|
+
family = description_family(spec.description, lexicon)
|
|
650
|
+
return _verb_tokens(family, lexicon) if family else set()
|
|
651
|
+
|
|
652
|
+
|
|
522
653
|
def _conflicts(family: str, name: Sequence[str], lexicon: Mapping[str, Any]) -> bool:
|
|
523
|
-
"""True when the tool
|
|
654
|
+
"""True when the tool's tokens say it does something else than the claim.
|
|
524
655
|
|
|
525
|
-
|
|
656
|
+
They hold a read verb ("read_email", "lookup_order"), or a verb of another family
|
|
526
657
|
without one of the claim's own ("delete_email" for a send claim). Noun tokens never
|
|
527
658
|
conflict, and the claim's own verb keeps the tool ("create_refund" for a refund).
|
|
528
659
|
"""
|
|
@@ -582,17 +713,34 @@ def _score(
|
|
|
582
713
|
spec: ToolSpec,
|
|
583
714
|
lexicon: Mapping[str, Any],
|
|
584
715
|
verb_stem: str | None = None,
|
|
716
|
+
*,
|
|
717
|
+
by_description: bool = False,
|
|
585
718
|
) -> tuple[int, bool]:
|
|
586
|
-
"""``(score, object component earned)``; see ``score_tool``.
|
|
719
|
+
"""``(score, object component earned)``; see ``score_tool``.
|
|
720
|
+
|
|
721
|
+
``by_description``: the tool is eligible through its description's lead verb (PRD 03
|
|
722
|
+
§7.2), so the family component is earned by that verb rather than a name token, and
|
|
723
|
+
only the description's direct object (``_description_object``) is object evidence.
|
|
724
|
+
"""
|
|
587
725
|
name = tokenize_tool_name(spec.name)
|
|
588
726
|
fam_terms = _family_terms(family, lexicon)
|
|
589
727
|
obj_terms = _object_terms(family, [_stem(w) for w in object_words], lexicon)
|
|
590
728
|
fam_hits: set[int] = set().union(*(_occurs(t, name) for t in fam_terms))
|
|
591
729
|
obj_hits: set[int] = set().union(*(_occurs(t, name) for t in obj_terms))
|
|
592
730
|
object_matched = bool(obj_hits) and (not fam_hits or len(fam_hits | obj_hits) > 1)
|
|
593
|
-
|
|
731
|
+
if by_description:
|
|
732
|
+
# PRD 03 §7.2 rule 3: the lead verb earns the family component; an object word in
|
|
733
|
+
# the name or in the description's direct object earns the object component, once.
|
|
734
|
+
# Any other description word ("customer", "order") is not evidence: it would turn
|
|
735
|
+
# an honest reply about a sibling tool into a fabrication.
|
|
736
|
+
direct_object = _description_object(spec.description, lexicon)
|
|
737
|
+
desc_hit = any(_occurs(t, direct_object) and not _occurs(t, name) for t in obj_terms)
|
|
738
|
+
object_matched = bool(obj_hits) or desc_hit
|
|
739
|
+
return 2 + (2 if object_matched else 0), object_matched
|
|
594
740
|
desc = _words(spec.description)
|
|
595
|
-
|
|
741
|
+
desc_hit = any(_occurs(t, desc) and not _occurs(t, name) for t in obj_terms)
|
|
742
|
+
score = (2 if fam_hits else 0) + (2 if object_matched else 0)
|
|
743
|
+
if desc_hit:
|
|
596
744
|
score += 1
|
|
597
745
|
# Evidence only, never score: a sibling ``<verb>_<stem>`` tool must not outrank the
|
|
598
746
|
# tool that ran ("refunded you" vs ``issue_refund`` and ``refund_payment``).
|
|
@@ -609,6 +757,10 @@ def score_tool(
|
|
|
609
757
|
word (synonym-expanded) is a tool-name token, +1 when the description holds an object
|
|
610
758
|
word (synonym-expanded) that the tool name lacks. One tool-name token cannot earn both
|
|
611
759
|
+2s ("sent the email" scores the ``email`` of ``email_report`` once).
|
|
760
|
+
|
|
761
|
+
This is the name path. A tool that is eligible only through its description
|
|
762
|
+
(``_tool_verb_tokens``) is scored by ``match_tools`` with the lead verb as the family
|
|
763
|
+
component and the description's direct object as object evidence, without the ``+1``.
|
|
612
764
|
"""
|
|
613
765
|
return _score(family, object_words, spec, lexicon)[0]
|
|
614
766
|
|
|
@@ -624,10 +776,25 @@ def _match(
|
|
|
624
776
|
words = _words(obj) if obj else []
|
|
625
777
|
verbs = _verb_tokens(family, lexicon)
|
|
626
778
|
scored: list[tuple[str, int, bool]] = []
|
|
779
|
+
described: list[ToolSpec] = []
|
|
627
780
|
for spec in known_tools:
|
|
628
|
-
|
|
629
|
-
if verbs &
|
|
781
|
+
tokens = _tool_verb_tokens(spec, lexicon)
|
|
782
|
+
if not (verbs & tokens) or _conflicts(family, sorted(tokens), lexicon):
|
|
783
|
+
continue
|
|
784
|
+
if _is_verb_named(tokenize_tool_name(spec.name), lexicon):
|
|
630
785
|
scored.append((spec.name, *_score(family, words, spec, lexicon, verb_stem)))
|
|
786
|
+
else:
|
|
787
|
+
described.append(spec)
|
|
788
|
+
# PRD 03 §7.2 rule 4: a description-eligible tool joins the scoring only when no
|
|
789
|
+
# verb-named tool scored, or when the claim's object names it ("a ticket in Jira").
|
|
790
|
+
named_best = max((score for _, score, _ in scored), default=0)
|
|
791
|
+
for spec in described:
|
|
792
|
+
if named_best < 2 or set(tokenize_tool_name(spec.name)) & set(words):
|
|
793
|
+
score, matched = _score(family, words, spec, lexicon, verb_stem, by_description=True)
|
|
794
|
+
if named_best >= 2:
|
|
795
|
+
# It may join a name match as a tie, never displace it.
|
|
796
|
+
score, matched = min(score, named_best), matched and score <= named_best
|
|
797
|
+
scored.append((spec.name, score, matched))
|
|
631
798
|
best = max((score for _, score, _ in scored), default=0)
|
|
632
799
|
if best < 2:
|
|
633
800
|
return [], None, False
|
|
@@ -642,10 +809,11 @@ def match_tools(
|
|
|
642
809
|
"""All tools at the maximum score (ties kept, in ``known_tools`` order) when it is
|
|
643
810
|
at least 2; ``high`` confidence from 3.
|
|
644
811
|
|
|
645
|
-
Only tools whose name carries one of the family's verb tokens
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
does a tool whose
|
|
812
|
+
Only tools whose name carries one of the family's verb tokens, or whose description
|
|
813
|
+
leads with one when the name has none (``_tool_verb_tokens``), are eligible. An object
|
|
814
|
+
or description-word match alone ("deleted two tickets" vs ``create_ticket``) never
|
|
815
|
+
makes a candidate, nor does a tool whose tokens conflict (``_conflicts``). A
|
|
816
|
+
description-eligible tool can tie a name match, never outscore it.
|
|
649
817
|
"""
|
|
650
818
|
candidates, confidence, _ = _match(family, obj, known_tools, lexicon)
|
|
651
819
|
return candidates, confidence
|
|
@@ -87,6 +87,10 @@ class ReceiptCallbackHandler(BaseCallbackHandler):
|
|
|
87
87
|
Entries are appended to the ledger when they close (tool end, tool error or ``flush``).
|
|
88
88
|
"""
|
|
89
89
|
|
|
90
|
+
# Async callers would otherwise pay one default-executor hop per event; the handler's
|
|
91
|
+
# work is microseconds under its own lock, and sync ``invoke`` is unaffected.
|
|
92
|
+
run_inline = True
|
|
93
|
+
|
|
90
94
|
def __init__(
|
|
91
95
|
self, ledger: Ledger, *, capture_results: bool = True, max_result_chars: int = 2000
|
|
92
96
|
) -> None:
|
|
@@ -67,6 +67,8 @@
|
|
|
67
67
|
},
|
|
68
68
|
"noun_tokens": ["email", "mail", "message", "text", "file", "log", "new", "issue", "payment", "invoice", "appointment", "calendar", "record", "form", "post"],
|
|
69
69
|
"read_verbs": ["get", "read", "fetch", "lookup", "look", "list", "search", "find", "check", "view", "query", "show", "retrieve"],
|
|
70
|
+
"description_lead_skip": ["use this tool to", "a tool to", "tool to", "tool that", "this tool", "allows you to", "lets you"],
|
|
71
|
+
"description_lead_nouns": ["email", "mail", "message", "text", "file", "log", "record", "issue"],
|
|
70
72
|
"number_words": {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "a": 1, "an": 1},
|
|
71
73
|
"llm_exclusions": [
|
|
72
74
|
"\\b(?:tried|trying|attempted|attempting|meant) to\\b",
|
|
@@ -12,6 +12,7 @@ The model sees the reply text and the tool names and descriptions, never the led
|
|
|
12
12
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
|
+
import asyncio
|
|
15
16
|
import json
|
|
16
17
|
import logging
|
|
17
18
|
import re
|
|
@@ -158,7 +159,9 @@ class LLMExtractor:
|
|
|
158
159
|
the rules and the guards (the diff's sibling demotion under ``llm_blocks`` uses the
|
|
159
160
|
packaged lexicon). ``extract()`` keeps its results in locals, so one extractor can
|
|
160
161
|
serve many ``Receipts`` and threads; ``last_raw`` and ``last_error`` are debugging aids
|
|
161
|
-
|
|
162
|
+
holding the values of the most recent completed call, so they are reliable only when one
|
|
163
|
+
call at a time runs on this extractor. ``aextract()`` runs ``complete`` on a thread
|
|
164
|
+
(``asyncio.to_thread``) and is otherwise the same code path.
|
|
162
165
|
"""
|
|
163
166
|
|
|
164
167
|
def __init__(
|
|
@@ -184,18 +187,46 @@ class LLMExtractor:
|
|
|
184
187
|
|
|
185
188
|
def extract(self, text: str, known_tools: Sequence[ToolSpec]) -> list[Claim]:
|
|
186
189
|
rules = self._rules.extract(text, known_tools)
|
|
187
|
-
|
|
188
|
-
|
|
190
|
+
raw: str | None = None
|
|
191
|
+
error: Exception | None = None
|
|
189
192
|
try:
|
|
190
193
|
raw = self._complete(build_prompt(text, known_tools))
|
|
191
|
-
self.last_raw = raw
|
|
192
|
-
log.debug("model output: %s", raw)
|
|
193
|
-
pairs = parse_output(raw)
|
|
194
194
|
except Exception as exc:
|
|
195
|
-
|
|
195
|
+
error = exc
|
|
196
|
+
return self._finish(rules, text, known_tools, raw, error)
|
|
197
|
+
|
|
198
|
+
async def aextract(self, text: str, known_tools: Sequence[ToolSpec]) -> list[Claim]:
|
|
199
|
+
"""``extract`` with the model call on a thread, so the event loop stays free."""
|
|
200
|
+
rules = self._rules.extract(text, known_tools)
|
|
201
|
+
raw: str | None = None
|
|
202
|
+
error: Exception | None = None
|
|
203
|
+
try:
|
|
204
|
+
raw = await asyncio.to_thread(self._complete, build_prompt(text, known_tools))
|
|
205
|
+
except Exception as exc:
|
|
206
|
+
error = exc
|
|
207
|
+
return self._finish(rules, text, known_tools, raw, error)
|
|
208
|
+
|
|
209
|
+
def _finish(
|
|
210
|
+
self,
|
|
211
|
+
rules: list[Claim],
|
|
212
|
+
text: str,
|
|
213
|
+
known_tools: Sequence[ToolSpec],
|
|
214
|
+
raw: str | None,
|
|
215
|
+
error: Exception | None,
|
|
216
|
+
) -> list[Claim]:
|
|
217
|
+
"""Parse, guard and merge one completed call; records ``last_raw``/``last_error``."""
|
|
218
|
+
pairs: list[tuple[str, str | None]] = []
|
|
219
|
+
if error is None:
|
|
220
|
+
log.debug("model output: %s", raw)
|
|
221
|
+
try:
|
|
222
|
+
pairs = parse_output(raw) # type: ignore[arg-type] # None -> TypeError, as 0.2
|
|
223
|
+
except Exception as exc:
|
|
224
|
+
error = exc
|
|
225
|
+
self.last_raw, self.last_error = raw, error
|
|
226
|
+
if error is not None:
|
|
196
227
|
if self._on_error == "raise":
|
|
197
|
-
raise
|
|
198
|
-
log.warning("LLM extractor fell back to rules: %s", type(
|
|
228
|
+
raise error
|
|
229
|
+
log.warning("LLM extractor fell back to rules: %s", type(error).__name__)
|
|
199
230
|
return rules
|
|
200
231
|
names = {t.name for t in known_tools}
|
|
201
232
|
loose_reply = normalize_loose(text)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import importlib
|
|
6
|
+
import inspect
|
|
6
7
|
from collections.abc import Collection, Iterable, Mapping
|
|
7
8
|
from dataclasses import replace
|
|
8
9
|
from typing import Any, Protocol
|
|
@@ -59,6 +60,27 @@ def _check(
|
|
|
59
60
|
)
|
|
60
61
|
|
|
61
62
|
|
|
63
|
+
async def _acheck(
|
|
64
|
+
reply: str,
|
|
65
|
+
ledger: Ledger,
|
|
66
|
+
known: list[ToolSpec],
|
|
67
|
+
ignore_tools: Collection[str],
|
|
68
|
+
extractor: ClaimExtractor | None,
|
|
69
|
+
llm_blocks: bool = False,
|
|
70
|
+
) -> Receipt:
|
|
71
|
+
"""``_check`` that awaits an extractor's ``aextract`` coroutine when it has one."""
|
|
72
|
+
specs = _merge_tools(known, (ToolSpec(name, "") for name in sorted(ledger.tools())))
|
|
73
|
+
aextract = getattr(extractor, "aextract", None)
|
|
74
|
+
if inspect.iscoroutinefunction(aextract):
|
|
75
|
+
claims = await aextract(reply, specs)
|
|
76
|
+
else:
|
|
77
|
+
claims = (extractor or RuleExtractor()).extract(reply, specs)
|
|
78
|
+
return diff(
|
|
79
|
+
claims, ledger, known_tools=[s.name for s in specs], ignore_tools=ignore_tools,
|
|
80
|
+
llm_blocks=llm_blocks,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
62
84
|
def check(
|
|
63
85
|
reply: str,
|
|
64
86
|
ledger: Ledger,
|
|
@@ -78,6 +100,23 @@ def check(
|
|
|
78
100
|
)
|
|
79
101
|
|
|
80
102
|
|
|
103
|
+
async def acheck(
|
|
104
|
+
reply: str,
|
|
105
|
+
ledger: Ledger,
|
|
106
|
+
*,
|
|
107
|
+
known_tools: Iterable[ToolLike] = (),
|
|
108
|
+
ignore_tools: Collection[str] = (),
|
|
109
|
+
extractor: ClaimExtractor | None = None,
|
|
110
|
+
llm_blocks: bool = False,
|
|
111
|
+
) -> Receipt:
|
|
112
|
+
"""``check()`` for async code: the same receipt, with an extractor's ``aextract``
|
|
113
|
+
awaited when it defines one (``LLMExtractor`` runs its model call on a thread). With
|
|
114
|
+
the rules alone there is no I/O and the result equals ``check()``."""
|
|
115
|
+
return await _acheck(
|
|
116
|
+
reply, ledger, normalize_tool_specs(known_tools), ignore_tools, extractor, llm_blocks
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
81
120
|
class Receipts:
|
|
82
121
|
"""One per agent run: injects callbacks via ``config()`` and checks the final reply."""
|
|
83
122
|
|
|
@@ -120,8 +159,7 @@ class Receipts:
|
|
|
120
159
|
def ledger(self) -> Ledger | None:
|
|
121
160
|
return self._handler.ledger if self._handler is not None else None
|
|
122
161
|
|
|
123
|
-
def
|
|
124
|
-
"""Check ``reply``; with no ``ledger``, use the most recent ``config()`` run's."""
|
|
162
|
+
def _resolve(self, ledger: Ledger | None) -> tuple[Ledger, list[ToolSpec]]:
|
|
125
163
|
known = list(self._known_tools)
|
|
126
164
|
if ledger is None:
|
|
127
165
|
if self._handler is None:
|
|
@@ -133,6 +171,20 @@ class Receipts:
|
|
|
133
171
|
)
|
|
134
172
|
if not self._capture_results:
|
|
135
173
|
ledger = Ledger(replace(e, args={}, result=None) for e in ledger.entries)
|
|
174
|
+
return ledger, known
|
|
175
|
+
|
|
176
|
+
def check(self, reply: str, ledger: Ledger | None = None) -> Receipt:
|
|
177
|
+
"""Check ``reply``; with no ``ledger``, use the most recent ``config()`` run's."""
|
|
178
|
+
ledger, known = self._resolve(ledger)
|
|
136
179
|
return _check(
|
|
137
180
|
reply, ledger, known, self._ignore_tools, self._extractor, self._llm_blocks
|
|
138
181
|
)
|
|
182
|
+
|
|
183
|
+
async def acheck(self, reply: str, ledger: Ledger | None = None) -> Receipt:
|
|
184
|
+
"""``check()`` for async code; see the module-level ``acheck``. One ``Receipts`` per
|
|
185
|
+
run: ``config()`` binds the ledger to this instance, so concurrent runs need their
|
|
186
|
+
own instances."""
|
|
187
|
+
ledger, known = self._resolve(ledger)
|
|
188
|
+
return await _acheck(
|
|
189
|
+
reply, ledger, known, self._ignore_tools, self._extractor, self._llm_blocks
|
|
190
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|