bantamkit 0.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bantamkit/__init__.py +32 -0
- bantamkit/agent.py +458 -0
- bantamkit/assets/contracts/default.yaml +90 -0
- bantamkit/assets/evals/devteam/manifest.yaml +351 -0
- bantamkit/assets/evals/devteam/repo/HISTORY.md +18 -0
- bantamkit/assets/evals/devteam/repo/README.md +12 -0
- bantamkit/assets/evals/devteam/repo/docs/architecture.md +17 -0
- bantamkit/assets/evals/devteam/repo/docs/runbook.md +10 -0
- bantamkit/assets/evals/devteam/repo/issues/142-settlement-timeout.md +23 -0
- bantamkit/assets/evals/devteam/repo/patches/0009-retry-budget.patch +38 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/__init__.py +3 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/config.py +35 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/errors.py +13 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/posting.py +12 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/registry.py +7 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/report.py +9 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/retry.py +17 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/settle.py +16 -0
- bantamkit/assets/evals/devteam/repo/src/ledger/validate.py +14 -0
- bantamkit/assets/evals/devteam/repo/tests/test_posting.py +13 -0
- bantamkit/assets/evals/devteam/repo/tests/test_settle.py +9 -0
- bantamkit/assets/evals/devteam/tasks/dt-error-contract.yaml +186 -0
- bantamkit/assets/evals/devteam/tasks/dt-handler-map.yaml +183 -0
- bantamkit/assets/evals/devteam/tasks/dt-patch-before-after.yaml +182 -0
- bantamkit/assets/evals/devteam/tasks/dt-retry-attempts.yaml +181 -0
- bantamkit/assets/evals/devteam/tasks/dt-settlement-config.yaml +185 -0
- bantamkit/assets/evals/devteam/tasks/dt-symbol-home.yaml +181 -0
- bantamkit/assets/evals/devteam/tasks/dt-trace-blame.yaml +182 -0
- bantamkit/assets/evals/devteam/tasks/dt-unread-key.yaml +180 -0
- bantamkit/assets/evals/document/tasks/doc-large-in-137.yaml +38 -0
- bantamkit/assets/evals/document/tasks/doc-large-in-359.yaml +44 -0
- bantamkit/assets/evals/document/tasks/doc-large-in-372.yaml +38 -0
- bantamkit/assets/evals/document/tasks/doc-large-out-11764.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-large-out-4137.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-large-out-8022.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-small-137.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-small-261.yaml +37 -0
- bantamkit/assets/evals/document/tasks/doc-small-388.yaml +37 -0
- bantamkit/assets/evals/fixtures/.gitkeep +0 -0
- bantamkit/assets/evals/fixtures/catalog.json +6 -0
- bantamkit/assets/evals/perturbations/task-completion.yaml +576 -0
- bantamkit/assets/evals/tasks/.gitkeep +0 -0
- bantamkit/assets/evals/tasks/extract-contact.yaml +14 -0
- bantamkit/assets/evals/tasks/extract-invoice.yaml +14 -0
- bantamkit/assets/evals/tasks/extract-order.yaml +15 -0
- bantamkit/assets/evals/tasks/extract-schedule.yaml +14 -0
- bantamkit/assets/evals/tasks/extract-versions.yaml +17 -0
- bantamkit/assets/evals/tasks/nav-prod-port.yaml +84 -0
- bantamkit/assets/evals/tasks/nav-release-bundle.yaml +87 -0
- bantamkit/assets/evals/tasks/recall-audit-retention.yaml +17 -0
- bantamkit/assets/evals/tasks/recall-cache-ttl.yaml +13 -0
- bantamkit/assets/evals/tasks/recall-db-port.yaml +17 -0
- bantamkit/assets/evals/tasks/recall-deploy.yaml +13 -0
- bantamkit/assets/evals/tasks/recall-env-endpoint.yaml +18 -0
- bantamkit/assets/evals/tasks/recall-oncall-rotation.yaml +21 -0
- bantamkit/assets/evals/tasks/recall-oncall.yaml +13 -0
- bantamkit/assets/evals/tasks/recall-org-quota.yaml +18 -0
- bantamkit/assets/evals/tasks/recall-owner.yaml +13 -0
- bantamkit/assets/evals/tasks/shop-basket-total.yaml +10 -0
- bantamkit/assets/evals/tasks/shop-cheapest.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-compare.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-gadget-value.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-stock-total.yaml +9 -0
- bantamkit/assets/evals/tasks/shop-total.yaml +9 -0
- bantamkit/assets/profiles/default.yaml +31 -0
- bantamkit/assets/profiles/patient.yaml +31 -0
- bantamkit/assets/rubrics/.gitkeep +0 -0
- bantamkit/assets/rubrics/code-quality.yaml +20 -0
- bantamkit/assets/rubrics/grounded-completion.yaml +37 -0
- bantamkit/assets/rubrics/task-completion.yaml +28 -0
- bantamkit/assets/schemas/shiftwork-checkpoint.json +188 -0
- bantamkit/assets/skills/.gitkeep +0 -0
- bantamkit/assets/skills/file-graph.md +7 -0
- bantamkit/assets/skills/memory.md +35 -0
- bantamkit/assets/tools/.gitkeep +0 -0
- bantamkit/assets/tools/bantamkit_read.json +48 -0
- bantamkit/assets/tools/bantamkit_status.json +25 -0
- bantamkit/assets/tools/build_identity.json +17 -0
- bantamkit/assets/tools/document_list.json +12 -0
- bantamkit/assets/tools/document_read.json +31 -0
- bantamkit/assets/tools/file_graph.json +12 -0
- bantamkit/assets/tools/memory_compact.json +31 -0
- bantamkit/assets/tools/memory_recall.json +38 -0
- bantamkit/assets/tools/memory_save.json +61 -0
- bantamkit/assets/tools/shiftwork_clock_in.json +25 -0
- bantamkit/assets/tools/shiftwork_clock_out.json +60 -0
- bantamkit/assets/tools/shiftwork_status.json +25 -0
- bantamkit/assets/tools/skill_audit.json +70 -0
- bantamkit/assets/tools/validate_json.json +31 -0
- bantamkit/assets.py +67 -0
- bantamkit/budget.py +114 -0
- bantamkit/client.py +329 -0
- bantamkit/contract.py +522 -0
- bantamkit/criticreplay.py +3241 -0
- bantamkit/critique.py +301 -0
- bantamkit/docread.py +1744 -0
- bantamkit/evalrun.py +2003 -0
- bantamkit/eventlog.py +282 -0
- bantamkit/filegraph.py +218 -0
- bantamkit/loopguard.py +101 -0
- bantamkit/mcpreport.py +763 -0
- bantamkit/mcpserver.py +1334 -0
- bantamkit/memory/__init__.py +28 -0
- bantamkit/memory/__main__.py +291 -0
- bantamkit/memory/component.py +569 -0
- bantamkit/memory/divergence.py +744 -0
- bantamkit/memory/layers.py +257 -0
- bantamkit/memory/store.py +940 -0
- bantamkit/pdfread.py +1402 -0
- bantamkit/profile.py +46 -0
- bantamkit/shiftwork.py +212 -0
- bantamkit/skillaudit.py +853 -0
- bantamkit/statusline.py +313 -0
- bantamkit/structured.py +125 -0
- bantamkit/textutil.py +30 -0
- bantamkit-0.27.0.dist-info/METADATA +207 -0
- bantamkit-0.27.0.dist-info/RECORD +119 -0
- bantamkit-0.27.0.dist-info/WHEEL +4 -0
- bantamkit-0.27.0.dist-info/entry_points.txt +2 -0
bantamkit/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from bantamkit.agent import Agent, AgentResult, MaxTurnsExceeded, ToolDef
|
|
2
|
+
from bantamkit.client import (
|
|
3
|
+
APIError,
|
|
4
|
+
BantamError,
|
|
5
|
+
Message,
|
|
6
|
+
ModelClient,
|
|
7
|
+
OpenAICompatible,
|
|
8
|
+
Response,
|
|
9
|
+
Tool,
|
|
10
|
+
ToolCall,
|
|
11
|
+
TransportError,
|
|
12
|
+
Usage,
|
|
13
|
+
)
|
|
14
|
+
from bantamkit.critique import (
|
|
15
|
+
CritiqueExhausted,
|
|
16
|
+
CritiqueGate,
|
|
17
|
+
GroundedCritiqueGate,
|
|
18
|
+
Rubric,
|
|
19
|
+
load_rubric,
|
|
20
|
+
render_evidence,
|
|
21
|
+
)
|
|
22
|
+
from bantamkit.evalrun import CONFIGS, format_report, run_suite
|
|
23
|
+
from bantamkit.filegraph import FileAccessGraph
|
|
24
|
+
from bantamkit.loopguard import LoopGuard
|
|
25
|
+
from bantamkit.memory import Memory, MemoryStore
|
|
26
|
+
from bantamkit.structured import StructuredOutputError, extract_json, structured
|
|
27
|
+
|
|
28
|
+
# RB-P45. The one place this checkout declares its version. `pyproject.toml` names this
|
|
29
|
+
# file as its dynamic version source, so the wheel's metadata and the string the MCP
|
|
30
|
+
# server advertises are the same committed bytes, and neither is a function of when
|
|
31
|
+
# someone last ran `pip`.
|
|
32
|
+
__version__ = "0.27.0"
|
bantamkit/agent.py
ADDED
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
"""Core agentic loop: chat → tool call → observe → repeat, within budgets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from contextlib import AbstractContextManager, ExitStack
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
|
|
10
|
+
from bantamkit.client import BantamError, Message, ModelClient, Tool, Usage
|
|
11
|
+
from bantamkit.contract import tool_argument_types, tool_arguments, tool_failed
|
|
12
|
+
from bantamkit.profile import default as profile_default
|
|
13
|
+
from bantamkit.textutil import truncate_counted
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def response_format_for(schema: dict) -> dict:
|
|
17
|
+
"""The wire shape of the constrained-decoding tier, in one place.
|
|
18
|
+
|
|
19
|
+
Lives here rather than in `structured.py` so both core callers — the one-shot
|
|
20
|
+
`structured()` loop and the agent loop — send byte-identical bodies; a second
|
|
21
|
+
spelling would make "same tier" unfalsifiable across the two paths.
|
|
22
|
+
"""
|
|
23
|
+
return {"type": "json_schema", "json_schema": {"name": "output", "schema": schema}}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _supports_response_format(client: ModelClient) -> bool:
|
|
27
|
+
"""Duck-typed capability check, same spirit as `seed`.
|
|
28
|
+
|
|
29
|
+
A client that understands the kwarg exposes the memo (`OpenAICompatible`
|
|
30
|
+
initializes it `False`); one that has met a 400 has flipped it to `True`.
|
|
31
|
+
Fake clients and adapters that never heard of the kwarg lack the attribute
|
|
32
|
+
entirely and are never sent it.
|
|
33
|
+
"""
|
|
34
|
+
return hasattr(client, "_response_format_unsupported") and not (
|
|
35
|
+
client._response_format_unsupported
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class MaxTurnsExceeded(BantamError):
|
|
40
|
+
"""The loop ended without a final answer inside the turn budget.
|
|
41
|
+
|
|
42
|
+
Carries the transcript up to the raise. Without it, the runs most worth
|
|
43
|
+
diagnosing are the ones that record nothing: v0.8.1's probes wrote
|
|
44
|
+
`messages: []` for every turns-exhausted run.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, message: str, messages: list[Message] | None = None):
|
|
48
|
+
super().__init__(message)
|
|
49
|
+
self.messages: list[Message] = list(messages or [])
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _attach_transcript(error: BantamError, messages: list[Message]) -> None:
|
|
53
|
+
"""Fill in a gate-raised error's transcript slot from the agent's own messages.
|
|
54
|
+
|
|
55
|
+
A post hook raises from inside `_first_feedback`, where the agent is the last
|
|
56
|
+
holder of the transcript — the hook was never handed one, and `run_task`'s own
|
|
57
|
+
`messages` list is still empty. Without this, every `schema-exhausted` and
|
|
58
|
+
`critique-exhausted` run recorded `messages: []`.
|
|
59
|
+
|
|
60
|
+
Opt-in by declaration, and fill-once: only an error that already carries a
|
|
61
|
+
`messages` list (the `MaxTurnsExceeded` shape) is filled, and only while that
|
|
62
|
+
list is empty. An error that declares no slot is left exactly as raised — a
|
|
63
|
+
transport failure is not a gate giving up, and measurement must not invent a
|
|
64
|
+
payload for it.
|
|
65
|
+
"""
|
|
66
|
+
existing = getattr(error, "messages", None)
|
|
67
|
+
if isinstance(existing, list) and not existing:
|
|
68
|
+
error.messages = list(messages)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class ToolDef:
|
|
73
|
+
tool: Tool
|
|
74
|
+
handler: Callable[..., str]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
_STRING_COERCIONS: dict[str, Callable[[str], object]] = {
|
|
78
|
+
"integer": int,
|
|
79
|
+
"number": float,
|
|
80
|
+
"boolean": lambda v: {"true": True, "false": False}[v.strip().lower()],
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def coerce_arguments(arguments: dict, parameters: dict) -> dict:
|
|
85
|
+
"""Adapt string-spelled scalars to the tool's declared parameter schema.
|
|
86
|
+
|
|
87
|
+
Small models send `k` as the JSON string `"10"` and the handler dies
|
|
88
|
+
comparing int to str — 17 of the 18 tool-argument failures in the 3b probe.
|
|
89
|
+
Only strings under an `integer`/`number`/`boolean` property are touched,
|
|
90
|
+
top level only; a conversion that fails passes the original value through
|
|
91
|
+
so today's error observation fires unchanged. Well-typed calls are no-ops.
|
|
92
|
+
"""
|
|
93
|
+
properties = parameters.get("properties") if isinstance(parameters, dict) else None
|
|
94
|
+
if not isinstance(properties, dict):
|
|
95
|
+
return arguments
|
|
96
|
+
coerced = dict(arguments)
|
|
97
|
+
for key, value in arguments.items():
|
|
98
|
+
schema = properties.get(key)
|
|
99
|
+
if not isinstance(value, str) or not isinstance(schema, dict):
|
|
100
|
+
continue
|
|
101
|
+
declared = schema.get("type")
|
|
102
|
+
convert = _STRING_COERCIONS.get(declared) if isinstance(declared, str) else None
|
|
103
|
+
if convert is None:
|
|
104
|
+
continue
|
|
105
|
+
try:
|
|
106
|
+
coerced[key] = convert(value)
|
|
107
|
+
except (ValueError, KeyError):
|
|
108
|
+
pass # unconvertible: hand the handler what the model actually sent
|
|
109
|
+
return coerced
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def declared_arguments(parameters: dict) -> list[str]:
|
|
113
|
+
"""The argument names a tool's schema declares, or [] if it declares no mapping."""
|
|
114
|
+
properties = parameters.get("properties") if isinstance(parameters, dict) else None
|
|
115
|
+
return sorted(properties) if isinstance(properties, dict) else []
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def select_declared_arguments(arguments: dict, parameters: dict) -> dict:
|
|
119
|
+
"""Drop the arguments a tool's schema does not declare, so an extra key cannot raise.
|
|
120
|
+
|
|
121
|
+
Measured need, 2026-08-20: on `document_list` — whose schema declares NO properties —
|
|
122
|
+
**5 of 5 seeds on the 4b** sent a spurious `document` argument. `handler(**arguments)`
|
|
123
|
+
raised `TypeError`, the model was told the tool had failed, and 3 of 4 smoke repeats then
|
|
124
|
+
answered that they were unable to access the workbook. A tool that crashes on a key it
|
|
125
|
+
never declared is measuring its own handler signature, not the model.
|
|
126
|
+
|
|
127
|
+
Ignoring the key rather than lecturing about it is the deliberate choice: an argument the
|
|
128
|
+
schema does not name is an argument the tool has no way to act on, so there is nothing to
|
|
129
|
+
negotiate, and the alternative spends one of ten turns telling the model something it
|
|
130
|
+
could have been shown by simply answering the call. Where the arguments genuinely cannot
|
|
131
|
+
be honoured — a declared one omitted — `_dispatch` still says so, in `contract`'s words.
|
|
132
|
+
|
|
133
|
+
Only a schema that declares a `properties` mapping and does not open itself with
|
|
134
|
+
`additionalProperties: true` is filtered. Anything else is passed through untouched,
|
|
135
|
+
because there the handler is the only thing that knows what it takes. Same shape and same
|
|
136
|
+
top-level-only scope as `coerce_arguments`, and a well-formed call is a no-op.
|
|
137
|
+
"""
|
|
138
|
+
if not isinstance(arguments, dict) or not isinstance(parameters, dict):
|
|
139
|
+
return arguments
|
|
140
|
+
if parameters.get("additionalProperties") is True:
|
|
141
|
+
return arguments
|
|
142
|
+
properties = parameters.get("properties")
|
|
143
|
+
if not isinstance(properties, dict):
|
|
144
|
+
return arguments
|
|
145
|
+
return {k: v for k, v in arguments.items() if k in properties}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
#: The JSON-Schema type names a tool may declare, mapped to what that type looks like once
|
|
149
|
+
#: the wire JSON has been parsed. `number` admits `int` because JSON Schema says an integer is
|
|
150
|
+
#: a number; `integer` does not admit `float`, because a model that sent `4137.0` for a row
|
|
151
|
+
#: offset is telling us something about its own arithmetic and a silent floor would hide it.
|
|
152
|
+
_JSON_TYPES: dict[str, tuple[type, ...]] = {
|
|
153
|
+
"string": (str,),
|
|
154
|
+
"integer": (int,),
|
|
155
|
+
"number": (int, float),
|
|
156
|
+
"boolean": (bool,),
|
|
157
|
+
"object": (dict,),
|
|
158
|
+
"array": (list, tuple),
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def json_type_of(value: object) -> str:
|
|
163
|
+
"""Name a sent value in the vocabulary the tool's own schema is written in.
|
|
164
|
+
|
|
165
|
+
The schema says `"type": "string"`; the model wrote JSON. Reporting `dict` would answer in
|
|
166
|
+
a third language — Python's — which is the language this whole defect was about. The
|
|
167
|
+
fallback is deliberately not `type(value).__name__`: an argument comes off a JSON wire and
|
|
168
|
+
is one of these seven, and a name Python invented is exactly what must not reach a model.
|
|
169
|
+
"""
|
|
170
|
+
if value is None:
|
|
171
|
+
return "null"
|
|
172
|
+
if isinstance(value, bool):
|
|
173
|
+
return "boolean"
|
|
174
|
+
if isinstance(value, int):
|
|
175
|
+
return "integer"
|
|
176
|
+
if isinstance(value, float):
|
|
177
|
+
return "number"
|
|
178
|
+
if isinstance(value, str):
|
|
179
|
+
return "string"
|
|
180
|
+
if isinstance(value, dict):
|
|
181
|
+
return "object"
|
|
182
|
+
if isinstance(value, (list, tuple)):
|
|
183
|
+
return "array"
|
|
184
|
+
return "unknown"
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def mistyped_arguments(arguments: dict, parameters: dict) -> list[tuple[str, str, str]]:
|
|
188
|
+
"""Which DECLARED arguments hold a value of a type the schema does not declare.
|
|
189
|
+
|
|
190
|
+
`RB-P86`, measured over J10's 432 graded runs. `select_declared_arguments` drops the keys
|
|
191
|
+
a schema does not name and type-checks nothing, so `llama3.2:3b` — which emits a
|
|
192
|
+
JSON-Schema FRAGMENT as the value of a declared parameter,
|
|
193
|
+
`{"document": {"description": "stock", "type": "string"}}` — had its dict passed straight
|
|
194
|
+
through to `docs.get(name)`, where a dict is not hashable. 13 observations of
|
|
195
|
+
`unhashable type: 'dict'`, all on `document_read`, all three `3b` reader cells
|
|
196
|
+
UNINFORMATIVE. That is the same class Amendment 1 fixed, one layer in.
|
|
197
|
+
|
|
198
|
+
REPORTED, not coerced and not dropped, and the three are genuinely different behaviours:
|
|
199
|
+
|
|
200
|
+
- **Coerced** is what `coerce_arguments` already does and this runs AFTER it, so the one
|
|
201
|
+
unambiguous conversion a small model actually needs — `"4137"` for a declared `integer`
|
|
202
|
+
— is already an `int` by the time anything here looks at it and is never reported. There
|
|
203
|
+
is no second conversion with that property. `{"type": "string"}` for a string has no
|
|
204
|
+
value to convert to, and `0` for a declared `string` is worse than unconvertible: to
|
|
205
|
+
`docread` an int part key is an INDEX and a str part key is a NAME, so `"0"` would ask a
|
|
206
|
+
different question than `0` did and the model would never be told the question changed.
|
|
207
|
+
- **Dropped** is right for an UNDECLARED key — the tool has no way to act on it, so there
|
|
208
|
+
is nothing to negotiate — and wrong here for the opposite reason: the schema names this
|
|
209
|
+
argument, so dropping it silently substitutes the handler's default. A model that asked
|
|
210
|
+
for `offset` as an object would be handed page 0 as though it had asked for page 0.
|
|
211
|
+
This repository already refuses that shape of answer (`document_offset_past_end` exists
|
|
212
|
+
because an empty page is "a dead end the model cannot tell from a real one").
|
|
213
|
+
- **Reported** costs one of ten turns and buys a sentence naming the argument, the type
|
|
214
|
+
the tool declares and the type that arrived. 11 of the 13 measured runs issued no
|
|
215
|
+
further tool call at all after reading CPython's sentence; a model cannot correct a
|
|
216
|
+
hash-table error, and can correct a sentence that names `offset` and the type its own
|
|
217
|
+
schema declares for it. The wording is the asset's; see `contract.tool_argument_types`.
|
|
218
|
+
|
|
219
|
+
Two exemptions, both deliberate. `null` is the wire spelling of "omitted" — every handler
|
|
220
|
+
in this repository already defaults its arguments to `None` and `_document_int` documents
|
|
221
|
+
None-as-fallback — so a null is passed through rather than reported. And an argument whose
|
|
222
|
+
schema declares no `type`, or a type name not in `_JSON_TYPES`, is not checked at all,
|
|
223
|
+
because there the schema has not said anything to hold the value against.
|
|
224
|
+
|
|
225
|
+
`True` is reported for a declared `integer` or `number` even though `isinstance(True, int)`
|
|
226
|
+
is Python-true: JSON Schema does not make a boolean a number, and `limit=true` is a model
|
|
227
|
+
losing track of its own call rather than asking for one row.
|
|
228
|
+
|
|
229
|
+
Same top-level-only scope as `coerce_arguments` and `select_declared_arguments`, and a
|
|
230
|
+
well-typed call is a no-op returning `[]`.
|
|
231
|
+
"""
|
|
232
|
+
if not isinstance(arguments, dict) or not isinstance(parameters, dict):
|
|
233
|
+
return []
|
|
234
|
+
properties = parameters.get("properties")
|
|
235
|
+
if not isinstance(properties, dict):
|
|
236
|
+
return []
|
|
237
|
+
wrong: list[tuple[str, str, str]] = []
|
|
238
|
+
for key in sorted(arguments):
|
|
239
|
+
value = arguments[key]
|
|
240
|
+
schema = properties.get(key)
|
|
241
|
+
if not isinstance(schema, dict) or value is None:
|
|
242
|
+
continue
|
|
243
|
+
declared = schema.get("type")
|
|
244
|
+
if not isinstance(declared, str):
|
|
245
|
+
continue
|
|
246
|
+
accepted = _JSON_TYPES.get(declared)
|
|
247
|
+
if accepted is None:
|
|
248
|
+
continue
|
|
249
|
+
numeric = declared in ("integer", "number")
|
|
250
|
+
if isinstance(value, accepted) and not (numeric and isinstance(value, bool)):
|
|
251
|
+
continue
|
|
252
|
+
wrong.append((key, declared, json_type_of(value)))
|
|
253
|
+
return wrong
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def handler_accepts(handler: Callable[..., str], arguments: dict) -> bool:
|
|
257
|
+
"""Can `handler` be CALLED with these arguments? Answered without calling it.
|
|
258
|
+
|
|
259
|
+
The point is the failure mode this replaces: a `TypeError` raised by the call itself
|
|
260
|
+
carries Python's own sentence, `qualname() missing 1 required positional argument`, and
|
|
261
|
+
that sentence used to be handed to the model verbatim. Binding the signature first turns
|
|
262
|
+
every argument-shaped failure into a contract string that names what the tool takes,
|
|
263
|
+
and leaves the `except` branch for exceptions raised INSIDE a handler.
|
|
264
|
+
|
|
265
|
+
A handler that cannot be introspected (a builtin, a C callable) is reported as accepting:
|
|
266
|
+
there the call is the only check that exists, and refusing it would be a guess.
|
|
267
|
+
"""
|
|
268
|
+
try:
|
|
269
|
+
signature = inspect.signature(handler)
|
|
270
|
+
except (TypeError, ValueError):
|
|
271
|
+
return True
|
|
272
|
+
try:
|
|
273
|
+
signature.bind(**arguments)
|
|
274
|
+
except TypeError:
|
|
275
|
+
return False
|
|
276
|
+
return True
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@dataclass
|
|
280
|
+
class AgentResult:
|
|
281
|
+
output: str
|
|
282
|
+
messages: list[Message]
|
|
283
|
+
usage: Usage
|
|
284
|
+
#: C-6, 2026-08-19. How many tool observations this run had cut to fit
|
|
285
|
+
#: `observation_budget`, and how many bytes went with them. The cut was always
|
|
286
|
+
#: announced IN BAND — `[truncated N bytes]` inside the string the model reads — and
|
|
287
|
+
#: nowhere else, so a run whose evidence was cut and one whose was not were the same
|
|
288
|
+
#: object to every caller, every scorer and every artifact. RB-P51: a run that
|
|
289
|
+
#: silently proceeded is not the same run as one that reported what it dropped.
|
|
290
|
+
#:
|
|
291
|
+
#: Zero on an uncut run, which is what keeps the column from being always-on. It is a
|
|
292
|
+
#: DISCLOSURE and not a verdict: the loop still proceeds, exactly as it did before,
|
|
293
|
+
#: and nothing here decides whether a cut run is scorable.
|
|
294
|
+
observations_truncated: int = 0
|
|
295
|
+
observation_bytes_dropped: int = 0
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass
|
|
299
|
+
class Agent:
|
|
300
|
+
client: ModelClient
|
|
301
|
+
tools: list[ToolDef] = field(default_factory=list)
|
|
302
|
+
system: str | None = None
|
|
303
|
+
max_turns: int | None = None
|
|
304
|
+
observation_budget: int | None = None
|
|
305
|
+
# Optional spend governor (`bantamkit.budget.TokenBudget`), attached by `use(...)`.
|
|
306
|
+
# Typed loosely and duck-typed at the call sites on purpose: core must not depend
|
|
307
|
+
# on the component, and an agent without one is byte-identical to before.
|
|
308
|
+
budget: object | None = None
|
|
309
|
+
# Optional constrained-decoding request for the loop's own model call
|
|
310
|
+
# (`response_format_for(schema)`). Unset by default and forwarded only to a client
|
|
311
|
+
# that understands the kwarg, so every existing caller stays byte-identical.
|
|
312
|
+
# It has to live on the loop, not on a gate: a gate only ever sees a violation that
|
|
313
|
+
# already happened, and the decode worth constraining is the first one.
|
|
314
|
+
response_format: dict | None = None
|
|
315
|
+
_post_hooks: list[Callable[..., str | None]] = field(default_factory=list)
|
|
316
|
+
# Component state that must be scoped to one assistant turn's tool calls rather
|
|
317
|
+
# than to the whole run — see `add_batch_scope`. An agent that registers none
|
|
318
|
+
# behaves exactly as it did before the hook existed.
|
|
319
|
+
_batch_scopes: list[Callable[[], AbstractContextManager]] = field(default_factory=list)
|
|
320
|
+
|
|
321
|
+
def __post_init__(self) -> None:
|
|
322
|
+
self.tools = list(self.tools or [])
|
|
323
|
+
if self.max_turns is None:
|
|
324
|
+
self.max_turns = profile_default("agent", "max_turns")
|
|
325
|
+
if self.observation_budget is None:
|
|
326
|
+
self.observation_budget = profile_default("agent", "observation_budget")
|
|
327
|
+
|
|
328
|
+
def use(self, *components) -> Agent:
|
|
329
|
+
for component in components:
|
|
330
|
+
component.setup(self)
|
|
331
|
+
return self
|
|
332
|
+
|
|
333
|
+
def register_tool(self, tooldef: ToolDef) -> None:
|
|
334
|
+
self.tools.append(tooldef)
|
|
335
|
+
|
|
336
|
+
def add_system(self, text: str) -> None:
|
|
337
|
+
self.system = f"{self.system}\n\n{text}" if self.system else text
|
|
338
|
+
|
|
339
|
+
def add_post_hook(self, hook: Callable[..., str | None]) -> None:
|
|
340
|
+
self._post_hooks.append(hook)
|
|
341
|
+
|
|
342
|
+
def add_batch_scope(self, scope: Callable[[], AbstractContextManager]) -> None:
|
|
343
|
+
"""Register a context-manager factory entered around each tool-call batch.
|
|
344
|
+
|
|
345
|
+
The model emits a whole batch of tool calls from one view of the world, and
|
|
346
|
+
this loop then dispatches them one after another — so a write early in the
|
|
347
|
+
batch is visible to a read later in the same batch, which the model has no
|
|
348
|
+
way to anticipate. A component whose state must not move under its own feet
|
|
349
|
+
mid-turn registers the boundary here; only the component knows what "not
|
|
350
|
+
moving" means for it, and only the loop knows where the turn ends.
|
|
351
|
+
"""
|
|
352
|
+
self._batch_scopes.append(scope)
|
|
353
|
+
|
|
354
|
+
def run(self, prompt: str) -> AgentResult:
|
|
355
|
+
messages: list[Message] = []
|
|
356
|
+
if self.system:
|
|
357
|
+
messages.append(Message(role="system", content=self.system))
|
|
358
|
+
messages.append(Message(role="user", content=prompt))
|
|
359
|
+
usage = Usage()
|
|
360
|
+
last_content = ""
|
|
361
|
+
observations_truncated = 0
|
|
362
|
+
observation_bytes_dropped = 0
|
|
363
|
+
|
|
364
|
+
for _ in range(self.max_turns):
|
|
365
|
+
if self.budget is not None and not self.budget.allow("required"):
|
|
366
|
+
# Past the hard ceiling. Stop iterating and hand back what the run
|
|
367
|
+
# already produced: a truncated answer is still scorable, and an
|
|
368
|
+
# exception here would convert a scorable answer into a loss.
|
|
369
|
+
return AgentResult(
|
|
370
|
+
output=last_content,
|
|
371
|
+
messages=messages,
|
|
372
|
+
usage=usage,
|
|
373
|
+
observations_truncated=observations_truncated,
|
|
374
|
+
observation_bytes_dropped=observation_bytes_dropped,
|
|
375
|
+
)
|
|
376
|
+
resp = self._chat(messages)
|
|
377
|
+
usage = usage + resp.usage
|
|
378
|
+
messages.append(resp.message)
|
|
379
|
+
if resp.message.content:
|
|
380
|
+
last_content = resp.message.content
|
|
381
|
+
|
|
382
|
+
if resp.message.tool_calls:
|
|
383
|
+
with ExitStack() as stack:
|
|
384
|
+
for scope in self._batch_scopes:
|
|
385
|
+
stack.enter_context(scope())
|
|
386
|
+
for tc in resp.message.tool_calls:
|
|
387
|
+
observation, dropped = truncate_counted(
|
|
388
|
+
self._dispatch(tc), self.observation_budget
|
|
389
|
+
)
|
|
390
|
+
if dropped:
|
|
391
|
+
observations_truncated += 1
|
|
392
|
+
observation_bytes_dropped += dropped
|
|
393
|
+
messages.append(
|
|
394
|
+
Message(role="tool", content=observation, tool_call_id=tc.id)
|
|
395
|
+
)
|
|
396
|
+
continue
|
|
397
|
+
|
|
398
|
+
output = resp.message.content or ""
|
|
399
|
+
try:
|
|
400
|
+
feedback = self._first_feedback(prompt, output, messages)
|
|
401
|
+
except BantamError as e:
|
|
402
|
+
# Recording only. The exception propagates unchanged — same type,
|
|
403
|
+
# same message, same traceback — so no verdict can move.
|
|
404
|
+
_attach_transcript(e, messages)
|
|
405
|
+
raise
|
|
406
|
+
if feedback is None:
|
|
407
|
+
return AgentResult(
|
|
408
|
+
output=output,
|
|
409
|
+
messages=messages,
|
|
410
|
+
usage=usage,
|
|
411
|
+
observations_truncated=observations_truncated,
|
|
412
|
+
observation_bytes_dropped=observation_bytes_dropped,
|
|
413
|
+
)
|
|
414
|
+
messages.append(Message(role="user", content=feedback))
|
|
415
|
+
|
|
416
|
+
raise MaxTurnsExceeded(f"no final answer within {self.max_turns} turns", messages)
|
|
417
|
+
|
|
418
|
+
def _chat(self, messages: list[Message]):
|
|
419
|
+
"""One model call, with the constrained-decoding tier when it is available.
|
|
420
|
+
|
|
421
|
+
Re-checked per turn, as `structured()` does: a 400 on turn 1 flips the client's
|
|
422
|
+
memo and silently drops the tier from there on.
|
|
423
|
+
"""
|
|
424
|
+
tools = [t.tool for t in self.tools] or None
|
|
425
|
+
if self.response_format is not None and _supports_response_format(self.client):
|
|
426
|
+
return self.client.chat(messages, tools=tools, response_format=self.response_format)
|
|
427
|
+
return self.client.chat(messages, tools=tools)
|
|
428
|
+
|
|
429
|
+
def _dispatch(self, tc) -> str:
|
|
430
|
+
tooldef = next((t for t in self.tools if t.tool.name == tc.name), None)
|
|
431
|
+
if tooldef is None:
|
|
432
|
+
names = [t.tool.name for t in self.tools]
|
|
433
|
+
return f"error: unknown tool '{tc.name}'. available tools: {names}"
|
|
434
|
+
try:
|
|
435
|
+
parameters = tooldef.tool.parameters
|
|
436
|
+
arguments = select_declared_arguments(
|
|
437
|
+
coerce_arguments(tc.arguments, parameters), parameters
|
|
438
|
+
)
|
|
439
|
+
mistyped = mistyped_arguments(arguments, parameters)
|
|
440
|
+
if mistyped:
|
|
441
|
+
return tool_argument_types(tc.name, mistyped)
|
|
442
|
+
if not handler_accepts(tooldef.handler, arguments):
|
|
443
|
+
return tool_arguments(tc.name, declared_arguments(parameters))
|
|
444
|
+
return str(tooldef.handler(**arguments))
|
|
445
|
+
except Exception as e:
|
|
446
|
+
return tool_failed(tc.name, e)
|
|
447
|
+
|
|
448
|
+
def _first_feedback(
|
|
449
|
+
self, task: str, output: str, messages: list[Message]
|
|
450
|
+
) -> str | None:
|
|
451
|
+
for hook in self._post_hooks:
|
|
452
|
+
if getattr(hook, "wants_transcript", False):
|
|
453
|
+
feedback = hook(task, output, messages)
|
|
454
|
+
else:
|
|
455
|
+
feedback = hook(task, output)
|
|
456
|
+
if feedback is not None:
|
|
457
|
+
return feedback
|
|
458
|
+
return None
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
name: default
|
|
2
|
+
schema_instruction: "Return ONLY a JSON object matching this JSON Schema. No prose.\n"
|
|
3
|
+
schema_retry: "{error}\nReturn ONLY a JSON object matching the schema."
|
|
4
|
+
critique_feedback: "A reviewer scored your answer {score}/10 (needs >= {threshold}). Feedback: {feedback}\nIf the feedback says a value is missing, unverified, or absent from your evidence, that is a fact you never looked up: call your tools and read the source that has it. Do not reword, generalise, or hedge the previous answer to work around the gap. Then answer again."
|
|
5
|
+
parse_error: "output was not parseable JSON: {detail}"
|
|
6
|
+
validation_error: "JSON does not match schema at '{where}': {detail}"
|
|
7
|
+
json_answer_retry: "Your answer contains no JSON. Restate your final answer as ONLY the JSON requested by the task, with no prose around it."
|
|
8
|
+
loop_note: "(you have now received this exact result {count} times; it will not change. Do something different or give your final answer now)"
|
|
9
|
+
loop_warn: "(STOP calling tools. Give your final answer now, in exactly the format the task asked for.)"
|
|
10
|
+
evidence_line: "{name}({arguments}) -> {observation}"
|
|
11
|
+
evidence_no_observation: "(no observation)"
|
|
12
|
+
evidence_empty: "(no tool calls were made)"
|
|
13
|
+
# The document reader pair's observations (`document_list`, `document_read`). Every string a
|
|
14
|
+
# reader hands back to the model lives here and not in the harness, for the reason the layer
|
|
15
|
+
# rule gives: the mechanics measured in `docread.py` transfer across models, this wording is
|
|
16
|
+
# exactly what does not.
|
|
17
|
+
document_manifest_empty: "no documents are attached to this task"
|
|
18
|
+
document_manifest_part: "{document} ({kind}) part {index} \"{part}\": {rows} rows, numbered 0 to {last}"
|
|
19
|
+
document_manifest_header_row: " row 0 is the header: {row}"
|
|
20
|
+
document_manifest_first_row: " row 1 is the first data row: {row}"
|
|
21
|
+
document_manifest_last_row: " row {index} is the last data row: {row}"
|
|
22
|
+
# What the rendering left out, stated in the one place that says what exists. Measured need
|
|
23
|
+
# (J25, the user's real ~/Downloads, 2026-08-20): `step test.xlsx` is 18.62 MB and the manifest
|
|
24
|
+
# said `4 parts, 28 rows` — all 28 of those rows are empty lines and the file's actual content
|
|
25
|
+
# is 56 embedded PNGs. Every line here is a COUNT: an adjective ("some images") tells a model
|
|
26
|
+
# nothing it can decide with, and silence tells it something false.
|
|
27
|
+
document_manifest_package_media: "{document}: the file also holds {count} embedded file(s) ({what}) totalling {bytes} bytes, which no row can carry — this reader renders no image or embedded object"
|
|
28
|
+
document_manifest_omitted_media: " NOT in those rows: {count} embedded file(s), {bytes} bytes, anchored to this part"
|
|
29
|
+
document_manifest_omitted_blank: " {count} of those {rows} rows carry no cell value at all and render as an empty line"
|
|
30
|
+
document_manifest_omitted_format: " column(s) {where}: {count} cell(s) store a NUMBER under the date/time format {what} — this reader renders the stored serial number verbatim and does not convert it to a date; convert it with that format code if you need one"
|
|
31
|
+
# J25-D3, the PDF reader. Two silences a page-addressable reader can produce that a
|
|
32
|
+
# spreadsheet cannot: a page that rendered no row (a scan, or glyphs no font maps), and
|
|
33
|
+
# characters dropped because the file states no mapping for them. An empty page and a page
|
|
34
|
+
# nobody could read are different facts and the model must be able to tell them apart.
|
|
35
|
+
document_manifest_omitted_unread_page: " this part rendered NO row: {why}. It draws {count} image(s), {bytes} bytes, that no row can carry — no row means this reader recovered no text from the page, which is NOT the same as the page being empty, and there is no OCR here"
|
|
36
|
+
document_manifest_unread_no_operator: "the page ran no text-showing operator at all, so nothing on it was ever text"
|
|
37
|
+
document_manifest_unread_unmapped: "the page ran {show_ops} text-showing operator(s) and every one of the {unmapped} character(s) they showed came through a font that declares no character map, so this reader dropped them rather than guess at them"
|
|
38
|
+
document_manifest_unread_whitespace: "the page ran {show_ops} text-showing operator(s) and this reader mapped every one of the {vouched} character(s) they showed, and every one of those characters is WHITESPACE — the text was recovered and it carries nothing, which is a fact about the page and not a failure of this reader"
|
|
39
|
+
document_manifest_unread_no_character: "the page ran {show_ops} text-showing operator(s) that put no character on the page at all, neither one this reader could map nor one it had to drop"
|
|
40
|
+
document_manifest_omitted_unmapped: " {count} character(s) shown on this part are NOT in those rows: their codes came from font(s) with no /ToUnicode map ({what}), so the file stores indices into a font's own glyphs and says nowhere which character each glyph draws — this reader drops them rather than guess, because a guess here is indistinguishable from content"
|
|
41
|
+
document_manifest_omitted_other: " NOT in those rows: {count} {subject} ({what})"
|
|
42
|
+
document_page_header: "{document} \"{part}\" rows {start}-{end} of {rows}; each line below begins with its own row number"
|
|
43
|
+
document_page_next: "more rows follow: call document_read again with offset={next_offset}"
|
|
44
|
+
document_page_end: "that was the last row of \"{part}\""
|
|
45
|
+
document_page_truncated: "row {index} was too long for one page and was cut: {dropped} bytes dropped"
|
|
46
|
+
document_unknown: "error: no document named {name}; this task has: {available}"
|
|
47
|
+
document_offset_past_end: "error: offset {offset} is past the end of \"{part}\", which has {rows} rows numbered 0 to {last}"
|
|
48
|
+
document_error: "error: {detail}"
|
|
49
|
+
# `bantamkit_read` (the MCP reader, job43) reuses every `document_*` sentence above with the
|
|
50
|
+
# path standing in for `{document}`. Two sentences are genuinely its own: its continuation
|
|
51
|
+
# line names ITS tool, and a part it does not have is a fact about the file, not the task.
|
|
52
|
+
bantamkit_read_page_next: "more rows follow: call bantamkit_read again with offset={next_offset}"
|
|
53
|
+
bantamkit_read_unknown_part: "error: no part named \"{part}\" in {path}; it has: {available}"
|
|
54
|
+
# The `paste` arm's system message (`docs/eval-data/2026-08-20-document-read-bar.md` §10.2,
|
|
55
|
+
# PASTE_MAX_BYTES = 8621 under Amendment 1, 2026-08-20). The arm registers NO tools, so these four lines are the whole of
|
|
56
|
+
# what the model is told ABOUT the corpus; the rows themselves are docread's own rendering,
|
|
57
|
+
# unprefixed and unreformatted, so `paste` and `document_read` differ in DELIVERY and not in
|
|
58
|
+
# content. Clause 4 makes stating the paste's own completeness part of the contract, and
|
|
59
|
+
# `paste_truncated` says the missing rows are unreachable rather than leaving the model to
|
|
60
|
+
# hunt for a tool that is not there: a paste that lets a truncated head read as the whole
|
|
61
|
+
# sheet is a different, worse arm than the one the bar pre-registered.
|
|
62
|
+
document_paste_preamble: "The following document content is attached to this task. Fields in a row are separated by tabs, rows are given in order, and row 0 of each part is its header."
|
|
63
|
+
document_paste_part: "{document} ({kind}) part {index} \"{part}\": {rows} rows, numbered 0 to {last}; {shown} of them are shown below."
|
|
64
|
+
document_paste_complete: " rows 0 to {last_shown} are shown, which is every row of \"{part}\": this copy is COMPLETE."
|
|
65
|
+
document_paste_truncated: " rows 0 to {last_shown} are shown; rows {first_missing} to {last} of \"{part}\" are NOT shown and no tool is attached that can fetch them. This copy is PARTIAL."
|
|
66
|
+
document_paste_none: " no rows of \"{part}\" are shown and no tool is attached that can fetch them."
|
|
67
|
+
# What the model is told when a tool CALL cannot be made or fails, rather than when a tool
|
|
68
|
+
# answers with its own error (`document_unknown` and friends above are that). It lives here
|
|
69
|
+
# for the reason the layer rule gives and for one measured reason: before 2026-08-20 the
|
|
70
|
+
# dispatcher formatted this sentence itself and interpolated the raw exception, so a model
|
|
71
|
+
# that sent one extra argument was handed
|
|
72
|
+
# `_document_tools.<locals>.list_documents() got an unexpected keyword argument 'document'`
|
|
73
|
+
# — a Python qualname, model-facing, owned by no asset and pinned by no golden. The
|
|
74
|
+
# argument-shaped failures are intercepted before the handler is called now, so `{detail}`
|
|
75
|
+
# carries a handler's OWN sentence and no longer a signature fragment.
|
|
76
|
+
tool_failed: "error: {tool} failed: {detail}. fix the arguments and retry."
|
|
77
|
+
tool_arguments: "error: {tool} does not take the arguments it was given. it takes: {accepts}. fix the arguments and retry."
|
|
78
|
+
tool_arguments_none: "error: {tool} takes no arguments at all. call it with none and retry."
|
|
79
|
+
# 2026-08-20, `RB-P86`. The sibling of the two above, one layer in. `select_declared_arguments`
|
|
80
|
+
# drops keys the schema does not declare and type-checks nothing, so `llama3.2:3b` — which
|
|
81
|
+
# emits a JSON-Schema FRAGMENT as the value of a declared parameter,
|
|
82
|
+
# `{"document": {"description": "stock", "type": "string"}}` — got its dict straight through to
|
|
83
|
+
# `docs.get(name)` and the model read `unhashable type: 'dict'`, thirteen times over 432 graded
|
|
84
|
+
# runs. That is CPython's sentence about a hash table, not a tool's sentence about its own
|
|
85
|
+
# arguments, and a 3b that receives it stops calling tools and answers that it cannot access
|
|
86
|
+
# the workbook. What it is told now names the argument, the type the tool declares and the type
|
|
87
|
+
# it was sent, so the next turn has something to correct. One frame for one or many, because
|
|
88
|
+
# the count is not the thing the model has to act on.
|
|
89
|
+
tool_argument_types: "error: {tool} was called with the wrong type of argument. {problems}. fix the arguments and retry."
|
|
90
|
+
tool_argument_type: "{argument} must be type {expected}, not type {actual}"
|