bantamkit 0.27.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. bantamkit/__init__.py +32 -0
  2. bantamkit/agent.py +458 -0
  3. bantamkit/assets/contracts/default.yaml +90 -0
  4. bantamkit/assets/evals/devteam/manifest.yaml +351 -0
  5. bantamkit/assets/evals/devteam/repo/HISTORY.md +18 -0
  6. bantamkit/assets/evals/devteam/repo/README.md +12 -0
  7. bantamkit/assets/evals/devteam/repo/docs/architecture.md +17 -0
  8. bantamkit/assets/evals/devteam/repo/docs/runbook.md +10 -0
  9. bantamkit/assets/evals/devteam/repo/issues/142-settlement-timeout.md +23 -0
  10. bantamkit/assets/evals/devteam/repo/patches/0009-retry-budget.patch +38 -0
  11. bantamkit/assets/evals/devteam/repo/src/ledger/__init__.py +3 -0
  12. bantamkit/assets/evals/devteam/repo/src/ledger/config.py +35 -0
  13. bantamkit/assets/evals/devteam/repo/src/ledger/errors.py +13 -0
  14. bantamkit/assets/evals/devteam/repo/src/ledger/posting.py +12 -0
  15. bantamkit/assets/evals/devteam/repo/src/ledger/registry.py +7 -0
  16. bantamkit/assets/evals/devteam/repo/src/ledger/report.py +9 -0
  17. bantamkit/assets/evals/devteam/repo/src/ledger/retry.py +17 -0
  18. bantamkit/assets/evals/devteam/repo/src/ledger/settle.py +16 -0
  19. bantamkit/assets/evals/devteam/repo/src/ledger/validate.py +14 -0
  20. bantamkit/assets/evals/devteam/repo/tests/test_posting.py +13 -0
  21. bantamkit/assets/evals/devteam/repo/tests/test_settle.py +9 -0
  22. bantamkit/assets/evals/devteam/tasks/dt-error-contract.yaml +186 -0
  23. bantamkit/assets/evals/devteam/tasks/dt-handler-map.yaml +183 -0
  24. bantamkit/assets/evals/devteam/tasks/dt-patch-before-after.yaml +182 -0
  25. bantamkit/assets/evals/devteam/tasks/dt-retry-attempts.yaml +181 -0
  26. bantamkit/assets/evals/devteam/tasks/dt-settlement-config.yaml +185 -0
  27. bantamkit/assets/evals/devteam/tasks/dt-symbol-home.yaml +181 -0
  28. bantamkit/assets/evals/devteam/tasks/dt-trace-blame.yaml +182 -0
  29. bantamkit/assets/evals/devteam/tasks/dt-unread-key.yaml +180 -0
  30. bantamkit/assets/evals/document/tasks/doc-large-in-137.yaml +38 -0
  31. bantamkit/assets/evals/document/tasks/doc-large-in-359.yaml +44 -0
  32. bantamkit/assets/evals/document/tasks/doc-large-in-372.yaml +38 -0
  33. bantamkit/assets/evals/document/tasks/doc-large-out-11764.yaml +37 -0
  34. bantamkit/assets/evals/document/tasks/doc-large-out-4137.yaml +37 -0
  35. bantamkit/assets/evals/document/tasks/doc-large-out-8022.yaml +37 -0
  36. bantamkit/assets/evals/document/tasks/doc-small-137.yaml +37 -0
  37. bantamkit/assets/evals/document/tasks/doc-small-261.yaml +37 -0
  38. bantamkit/assets/evals/document/tasks/doc-small-388.yaml +37 -0
  39. bantamkit/assets/evals/fixtures/.gitkeep +0 -0
  40. bantamkit/assets/evals/fixtures/catalog.json +6 -0
  41. bantamkit/assets/evals/perturbations/task-completion.yaml +576 -0
  42. bantamkit/assets/evals/tasks/.gitkeep +0 -0
  43. bantamkit/assets/evals/tasks/extract-contact.yaml +14 -0
  44. bantamkit/assets/evals/tasks/extract-invoice.yaml +14 -0
  45. bantamkit/assets/evals/tasks/extract-order.yaml +15 -0
  46. bantamkit/assets/evals/tasks/extract-schedule.yaml +14 -0
  47. bantamkit/assets/evals/tasks/extract-versions.yaml +17 -0
  48. bantamkit/assets/evals/tasks/nav-prod-port.yaml +84 -0
  49. bantamkit/assets/evals/tasks/nav-release-bundle.yaml +87 -0
  50. bantamkit/assets/evals/tasks/recall-audit-retention.yaml +17 -0
  51. bantamkit/assets/evals/tasks/recall-cache-ttl.yaml +13 -0
  52. bantamkit/assets/evals/tasks/recall-db-port.yaml +17 -0
  53. bantamkit/assets/evals/tasks/recall-deploy.yaml +13 -0
  54. bantamkit/assets/evals/tasks/recall-env-endpoint.yaml +18 -0
  55. bantamkit/assets/evals/tasks/recall-oncall-rotation.yaml +21 -0
  56. bantamkit/assets/evals/tasks/recall-oncall.yaml +13 -0
  57. bantamkit/assets/evals/tasks/recall-org-quota.yaml +18 -0
  58. bantamkit/assets/evals/tasks/recall-owner.yaml +13 -0
  59. bantamkit/assets/evals/tasks/shop-basket-total.yaml +10 -0
  60. bantamkit/assets/evals/tasks/shop-cheapest.yaml +9 -0
  61. bantamkit/assets/evals/tasks/shop-compare.yaml +9 -0
  62. bantamkit/assets/evals/tasks/shop-gadget-value.yaml +9 -0
  63. bantamkit/assets/evals/tasks/shop-stock-total.yaml +9 -0
  64. bantamkit/assets/evals/tasks/shop-total.yaml +9 -0
  65. bantamkit/assets/profiles/default.yaml +31 -0
  66. bantamkit/assets/profiles/patient.yaml +31 -0
  67. bantamkit/assets/rubrics/.gitkeep +0 -0
  68. bantamkit/assets/rubrics/code-quality.yaml +20 -0
  69. bantamkit/assets/rubrics/grounded-completion.yaml +37 -0
  70. bantamkit/assets/rubrics/task-completion.yaml +28 -0
  71. bantamkit/assets/schemas/shiftwork-checkpoint.json +188 -0
  72. bantamkit/assets/skills/.gitkeep +0 -0
  73. bantamkit/assets/skills/file-graph.md +7 -0
  74. bantamkit/assets/skills/memory.md +35 -0
  75. bantamkit/assets/tools/.gitkeep +0 -0
  76. bantamkit/assets/tools/bantamkit_read.json +48 -0
  77. bantamkit/assets/tools/bantamkit_status.json +25 -0
  78. bantamkit/assets/tools/build_identity.json +17 -0
  79. bantamkit/assets/tools/document_list.json +12 -0
  80. bantamkit/assets/tools/document_read.json +31 -0
  81. bantamkit/assets/tools/file_graph.json +12 -0
  82. bantamkit/assets/tools/memory_compact.json +31 -0
  83. bantamkit/assets/tools/memory_recall.json +38 -0
  84. bantamkit/assets/tools/memory_save.json +61 -0
  85. bantamkit/assets/tools/shiftwork_clock_in.json +25 -0
  86. bantamkit/assets/tools/shiftwork_clock_out.json +60 -0
  87. bantamkit/assets/tools/shiftwork_status.json +25 -0
  88. bantamkit/assets/tools/skill_audit.json +70 -0
  89. bantamkit/assets/tools/validate_json.json +31 -0
  90. bantamkit/assets.py +67 -0
  91. bantamkit/budget.py +114 -0
  92. bantamkit/client.py +329 -0
  93. bantamkit/contract.py +522 -0
  94. bantamkit/criticreplay.py +3241 -0
  95. bantamkit/critique.py +301 -0
  96. bantamkit/docread.py +1744 -0
  97. bantamkit/evalrun.py +2003 -0
  98. bantamkit/eventlog.py +282 -0
  99. bantamkit/filegraph.py +218 -0
  100. bantamkit/loopguard.py +101 -0
  101. bantamkit/mcpreport.py +763 -0
  102. bantamkit/mcpserver.py +1334 -0
  103. bantamkit/memory/__init__.py +28 -0
  104. bantamkit/memory/__main__.py +291 -0
  105. bantamkit/memory/component.py +569 -0
  106. bantamkit/memory/divergence.py +744 -0
  107. bantamkit/memory/layers.py +257 -0
  108. bantamkit/memory/store.py +940 -0
  109. bantamkit/pdfread.py +1402 -0
  110. bantamkit/profile.py +46 -0
  111. bantamkit/shiftwork.py +212 -0
  112. bantamkit/skillaudit.py +853 -0
  113. bantamkit/statusline.py +313 -0
  114. bantamkit/structured.py +125 -0
  115. bantamkit/textutil.py +30 -0
  116. bantamkit-0.27.0.dist-info/METADATA +207 -0
  117. bantamkit-0.27.0.dist-info/RECORD +119 -0
  118. bantamkit-0.27.0.dist-info/WHEEL +4 -0
  119. bantamkit-0.27.0.dist-info/entry_points.txt +2 -0
bantamkit/__init__.py ADDED
@@ -0,0 +1,32 @@
1
+ from bantamkit.agent import Agent, AgentResult, MaxTurnsExceeded, ToolDef
2
+ from bantamkit.client import (
3
+ APIError,
4
+ BantamError,
5
+ Message,
6
+ ModelClient,
7
+ OpenAICompatible,
8
+ Response,
9
+ Tool,
10
+ ToolCall,
11
+ TransportError,
12
+ Usage,
13
+ )
14
+ from bantamkit.critique import (
15
+ CritiqueExhausted,
16
+ CritiqueGate,
17
+ GroundedCritiqueGate,
18
+ Rubric,
19
+ load_rubric,
20
+ render_evidence,
21
+ )
22
+ from bantamkit.evalrun import CONFIGS, format_report, run_suite
23
+ from bantamkit.filegraph import FileAccessGraph
24
+ from bantamkit.loopguard import LoopGuard
25
+ from bantamkit.memory import Memory, MemoryStore
26
+ from bantamkit.structured import StructuredOutputError, extract_json, structured
27
+
28
+ # RB-P45. The one place this checkout declares its version. `pyproject.toml` names this
29
+ # file as its dynamic version source, so the wheel's metadata and the string the MCP
30
+ # server advertises are the same committed bytes, and neither is a function of when
31
+ # someone last ran `pip`.
32
+ __version__ = "0.27.0"
bantamkit/agent.py ADDED
@@ -0,0 +1,458 @@
1
+ """Core agentic loop: chat → tool call → observe → repeat, within budgets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import inspect
6
+ from collections.abc import Callable
7
+ from contextlib import AbstractContextManager, ExitStack
8
+ from dataclasses import dataclass, field
9
+
10
+ from bantamkit.client import BantamError, Message, ModelClient, Tool, Usage
11
+ from bantamkit.contract import tool_argument_types, tool_arguments, tool_failed
12
+ from bantamkit.profile import default as profile_default
13
+ from bantamkit.textutil import truncate_counted
14
+
15
+
16
+ def response_format_for(schema: dict) -> dict:
17
+ """The wire shape of the constrained-decoding tier, in one place.
18
+
19
+ Lives here rather than in `structured.py` so both core callers — the one-shot
20
+ `structured()` loop and the agent loop — send byte-identical bodies; a second
21
+ spelling would make "same tier" unfalsifiable across the two paths.
22
+ """
23
+ return {"type": "json_schema", "json_schema": {"name": "output", "schema": schema}}
24
+
25
+
26
+ def _supports_response_format(client: ModelClient) -> bool:
27
+ """Duck-typed capability check, same spirit as `seed`.
28
+
29
+ A client that understands the kwarg exposes the memo (`OpenAICompatible`
30
+ initializes it `False`); one that has met a 400 has flipped it to `True`.
31
+ Fake clients and adapters that never heard of the kwarg lack the attribute
32
+ entirely and are never sent it.
33
+ """
34
+ return hasattr(client, "_response_format_unsupported") and not (
35
+ client._response_format_unsupported
36
+ )
37
+
38
+
39
+ class MaxTurnsExceeded(BantamError):
40
+ """The loop ended without a final answer inside the turn budget.
41
+
42
+ Carries the transcript up to the raise. Without it, the runs most worth
43
+ diagnosing are the ones that record nothing: v0.8.1's probes wrote
44
+ `messages: []` for every turns-exhausted run.
45
+ """
46
+
47
+ def __init__(self, message: str, messages: list[Message] | None = None):
48
+ super().__init__(message)
49
+ self.messages: list[Message] = list(messages or [])
50
+
51
+
52
+ def _attach_transcript(error: BantamError, messages: list[Message]) -> None:
53
+ """Fill in a gate-raised error's transcript slot from the agent's own messages.
54
+
55
+ A post hook raises from inside `_first_feedback`, where the agent is the last
56
+ holder of the transcript — the hook was never handed one, and `run_task`'s own
57
+ `messages` list is still empty. Without this, every `schema-exhausted` and
58
+ `critique-exhausted` run recorded `messages: []`.
59
+
60
+ Opt-in by declaration, and fill-once: only an error that already carries a
61
+ `messages` list (the `MaxTurnsExceeded` shape) is filled, and only while that
62
+ list is empty. An error that declares no slot is left exactly as raised — a
63
+ transport failure is not a gate giving up, and measurement must not invent a
64
+ payload for it.
65
+ """
66
+ existing = getattr(error, "messages", None)
67
+ if isinstance(existing, list) and not existing:
68
+ error.messages = list(messages)
69
+
70
+
71
+ @dataclass
72
+ class ToolDef:
73
+ tool: Tool
74
+ handler: Callable[..., str]
75
+
76
+
77
+ _STRING_COERCIONS: dict[str, Callable[[str], object]] = {
78
+ "integer": int,
79
+ "number": float,
80
+ "boolean": lambda v: {"true": True, "false": False}[v.strip().lower()],
81
+ }
82
+
83
+
84
+ def coerce_arguments(arguments: dict, parameters: dict) -> dict:
85
+ """Adapt string-spelled scalars to the tool's declared parameter schema.
86
+
87
+ Small models send `k` as the JSON string `"10"` and the handler dies
88
+ comparing int to str — 17 of the 18 tool-argument failures in the 3b probe.
89
+ Only strings under an `integer`/`number`/`boolean` property are touched,
90
+ top level only; a conversion that fails passes the original value through
91
+ so today's error observation fires unchanged. Well-typed calls are no-ops.
92
+ """
93
+ properties = parameters.get("properties") if isinstance(parameters, dict) else None
94
+ if not isinstance(properties, dict):
95
+ return arguments
96
+ coerced = dict(arguments)
97
+ for key, value in arguments.items():
98
+ schema = properties.get(key)
99
+ if not isinstance(value, str) or not isinstance(schema, dict):
100
+ continue
101
+ declared = schema.get("type")
102
+ convert = _STRING_COERCIONS.get(declared) if isinstance(declared, str) else None
103
+ if convert is None:
104
+ continue
105
+ try:
106
+ coerced[key] = convert(value)
107
+ except (ValueError, KeyError):
108
+ pass # unconvertible: hand the handler what the model actually sent
109
+ return coerced
110
+
111
+
112
+ def declared_arguments(parameters: dict) -> list[str]:
113
+ """The argument names a tool's schema declares, or [] if it declares no mapping."""
114
+ properties = parameters.get("properties") if isinstance(parameters, dict) else None
115
+ return sorted(properties) if isinstance(properties, dict) else []
116
+
117
+
118
+ def select_declared_arguments(arguments: dict, parameters: dict) -> dict:
119
+ """Drop the arguments a tool's schema does not declare, so an extra key cannot raise.
120
+
121
+ Measured need, 2026-08-20: on `document_list` — whose schema declares NO properties —
122
+ **5 of 5 seeds on the 4b** sent a spurious `document` argument. `handler(**arguments)`
123
+ raised `TypeError`, the model was told the tool had failed, and 3 of 4 smoke repeats then
124
+ answered that they were unable to access the workbook. A tool that crashes on a key it
125
+ never declared is measuring its own handler signature, not the model.
126
+
127
+ Ignoring the key rather than lecturing about it is the deliberate choice: an argument the
128
+ schema does not name is an argument the tool has no way to act on, so there is nothing to
129
+ negotiate, and the alternative spends one of ten turns telling the model something it
130
+ could have been shown by simply answering the call. Where the arguments genuinely cannot
131
+ be honoured — a declared one omitted — `_dispatch` still says so, in `contract`'s words.
132
+
133
+ Only a schema that declares a `properties` mapping and does not open itself with
134
+ `additionalProperties: true` is filtered. Anything else is passed through untouched,
135
+ because there the handler is the only thing that knows what it takes. Same shape and same
136
+ top-level-only scope as `coerce_arguments`, and a well-formed call is a no-op.
137
+ """
138
+ if not isinstance(arguments, dict) or not isinstance(parameters, dict):
139
+ return arguments
140
+ if parameters.get("additionalProperties") is True:
141
+ return arguments
142
+ properties = parameters.get("properties")
143
+ if not isinstance(properties, dict):
144
+ return arguments
145
+ return {k: v for k, v in arguments.items() if k in properties}
146
+
147
+
148
+ #: The JSON-Schema type names a tool may declare, mapped to what that type looks like once
149
+ #: the wire JSON has been parsed. `number` admits `int` because JSON Schema says an integer is
150
+ #: a number; `integer` does not admit `float`, because a model that sent `4137.0` for a row
151
+ #: offset is telling us something about its own arithmetic and a silent floor would hide it.
152
+ _JSON_TYPES: dict[str, tuple[type, ...]] = {
153
+ "string": (str,),
154
+ "integer": (int,),
155
+ "number": (int, float),
156
+ "boolean": (bool,),
157
+ "object": (dict,),
158
+ "array": (list, tuple),
159
+ }
160
+
161
+
162
+ def json_type_of(value: object) -> str:
163
+ """Name a sent value in the vocabulary the tool's own schema is written in.
164
+
165
+ The schema says `"type": "string"`; the model wrote JSON. Reporting `dict` would answer in
166
+ a third language — Python's — which is the language this whole defect was about. The
167
+ fallback is deliberately not `type(value).__name__`: an argument comes off a JSON wire and
168
+ is one of these seven, and a name Python invented is exactly what must not reach a model.
169
+ """
170
+ if value is None:
171
+ return "null"
172
+ if isinstance(value, bool):
173
+ return "boolean"
174
+ if isinstance(value, int):
175
+ return "integer"
176
+ if isinstance(value, float):
177
+ return "number"
178
+ if isinstance(value, str):
179
+ return "string"
180
+ if isinstance(value, dict):
181
+ return "object"
182
+ if isinstance(value, (list, tuple)):
183
+ return "array"
184
+ return "unknown"
185
+
186
+
187
+ def mistyped_arguments(arguments: dict, parameters: dict) -> list[tuple[str, str, str]]:
188
+ """Which DECLARED arguments hold a value of a type the schema does not declare.
189
+
190
+ `RB-P86`, measured over J10's 432 graded runs. `select_declared_arguments` drops the keys
191
+ a schema does not name and type-checks nothing, so `llama3.2:3b` — which emits a
192
+ JSON-Schema FRAGMENT as the value of a declared parameter,
193
+ `{"document": {"description": "stock", "type": "string"}}` — had its dict passed straight
194
+ through to `docs.get(name)`, where a dict is not hashable. 13 observations of
195
+ `unhashable type: 'dict'`, all on `document_read`, all three `3b` reader cells
196
+ UNINFORMATIVE. That is the same class Amendment 1 fixed, one layer in.
197
+
198
+ REPORTED, not coerced and not dropped, and the three are genuinely different behaviours:
199
+
200
+ - **Coerced** is what `coerce_arguments` already does and this runs AFTER it, so the one
201
+ unambiguous conversion a small model actually needs — `"4137"` for a declared `integer`
202
+ — is already an `int` by the time anything here looks at it and is never reported. There
203
+ is no second conversion with that property. `{"type": "string"}` for a string has no
204
+ value to convert to, and `0` for a declared `string` is worse than unconvertible: to
205
+ `docread` an int part key is an INDEX and a str part key is a NAME, so `"0"` would ask a
206
+ different question than `0` did and the model would never be told the question changed.
207
+ - **Dropped** is right for an UNDECLARED key — the tool has no way to act on it, so there
208
+ is nothing to negotiate — and wrong here for the opposite reason: the schema names this
209
+ argument, so dropping it silently substitutes the handler's default. A model that asked
210
+ for `offset` as an object would be handed page 0 as though it had asked for page 0.
211
+ This repository already refuses that shape of answer (`document_offset_past_end` exists
212
+ because an empty page is "a dead end the model cannot tell from a real one").
213
+ - **Reported** costs one of ten turns and buys a sentence naming the argument, the type
214
+ the tool declares and the type that arrived. 11 of the 13 measured runs issued no
215
+ further tool call at all after reading CPython's sentence; a model cannot correct a
216
+ hash-table error, and can correct a sentence that names `offset` and the type its own
217
+ schema declares for it. The wording is the asset's; see `contract.tool_argument_types`.
218
+
219
+ Two exemptions, both deliberate. `null` is the wire spelling of "omitted" — every handler
220
+ in this repository already defaults its arguments to `None` and `_document_int` documents
221
+ None-as-fallback — so a null is passed through rather than reported. And an argument whose
222
+ schema declares no `type`, or a type name not in `_JSON_TYPES`, is not checked at all,
223
+ because there the schema has not said anything to hold the value against.
224
+
225
+ `True` is reported for a declared `integer` or `number` even though `isinstance(True, int)`
226
+ is Python-true: JSON Schema does not make a boolean a number, and `limit=true` is a model
227
+ losing track of its own call rather than asking for one row.
228
+
229
+ Same top-level-only scope as `coerce_arguments` and `select_declared_arguments`, and a
230
+ well-typed call is a no-op returning `[]`.
231
+ """
232
+ if not isinstance(arguments, dict) or not isinstance(parameters, dict):
233
+ return []
234
+ properties = parameters.get("properties")
235
+ if not isinstance(properties, dict):
236
+ return []
237
+ wrong: list[tuple[str, str, str]] = []
238
+ for key in sorted(arguments):
239
+ value = arguments[key]
240
+ schema = properties.get(key)
241
+ if not isinstance(schema, dict) or value is None:
242
+ continue
243
+ declared = schema.get("type")
244
+ if not isinstance(declared, str):
245
+ continue
246
+ accepted = _JSON_TYPES.get(declared)
247
+ if accepted is None:
248
+ continue
249
+ numeric = declared in ("integer", "number")
250
+ if isinstance(value, accepted) and not (numeric and isinstance(value, bool)):
251
+ continue
252
+ wrong.append((key, declared, json_type_of(value)))
253
+ return wrong
254
+
255
+
256
+ def handler_accepts(handler: Callable[..., str], arguments: dict) -> bool:
257
+ """Can `handler` be CALLED with these arguments? Answered without calling it.
258
+
259
+ The point is the failure mode this replaces: a `TypeError` raised by the call itself
260
+ carries Python's own sentence, `qualname() missing 1 required positional argument`, and
261
+ that sentence used to be handed to the model verbatim. Binding the signature first turns
262
+ every argument-shaped failure into a contract string that names what the tool takes,
263
+ and leaves the `except` branch for exceptions raised INSIDE a handler.
264
+
265
+ A handler that cannot be introspected (a builtin, a C callable) is reported as accepting:
266
+ there the call is the only check that exists, and refusing it would be a guess.
267
+ """
268
+ try:
269
+ signature = inspect.signature(handler)
270
+ except (TypeError, ValueError):
271
+ return True
272
+ try:
273
+ signature.bind(**arguments)
274
+ except TypeError:
275
+ return False
276
+ return True
277
+
278
+
279
+ @dataclass
280
+ class AgentResult:
281
+ output: str
282
+ messages: list[Message]
283
+ usage: Usage
284
+ #: C-6, 2026-08-19. How many tool observations this run had cut to fit
285
+ #: `observation_budget`, and how many bytes went with them. The cut was always
286
+ #: announced IN BAND — `[truncated N bytes]` inside the string the model reads — and
287
+ #: nowhere else, so a run whose evidence was cut and one whose was not were the same
288
+ #: object to every caller, every scorer and every artifact. RB-P51: a run that
289
+ #: silently proceeded is not the same run as one that reported what it dropped.
290
+ #:
291
+ #: Zero on an uncut run, which is what keeps the column from being always-on. It is a
292
+ #: DISCLOSURE and not a verdict: the loop still proceeds, exactly as it did before,
293
+ #: and nothing here decides whether a cut run is scorable.
294
+ observations_truncated: int = 0
295
+ observation_bytes_dropped: int = 0
296
+
297
+
298
+ @dataclass
299
+ class Agent:
300
+ client: ModelClient
301
+ tools: list[ToolDef] = field(default_factory=list)
302
+ system: str | None = None
303
+ max_turns: int | None = None
304
+ observation_budget: int | None = None
305
+ # Optional spend governor (`bantamkit.budget.TokenBudget`), attached by `use(...)`.
306
+ # Typed loosely and duck-typed at the call sites on purpose: core must not depend
307
+ # on the component, and an agent without one is byte-identical to before.
308
+ budget: object | None = None
309
+ # Optional constrained-decoding request for the loop's own model call
310
+ # (`response_format_for(schema)`). Unset by default and forwarded only to a client
311
+ # that understands the kwarg, so every existing caller stays byte-identical.
312
+ # It has to live on the loop, not on a gate: a gate only ever sees a violation that
313
+ # already happened, and the decode worth constraining is the first one.
314
+ response_format: dict | None = None
315
+ _post_hooks: list[Callable[..., str | None]] = field(default_factory=list)
316
+ # Component state that must be scoped to one assistant turn's tool calls rather
317
+ # than to the whole run — see `add_batch_scope`. An agent that registers none
318
+ # behaves exactly as it did before the hook existed.
319
+ _batch_scopes: list[Callable[[], AbstractContextManager]] = field(default_factory=list)
320
+
321
+ def __post_init__(self) -> None:
322
+ self.tools = list(self.tools or [])
323
+ if self.max_turns is None:
324
+ self.max_turns = profile_default("agent", "max_turns")
325
+ if self.observation_budget is None:
326
+ self.observation_budget = profile_default("agent", "observation_budget")
327
+
328
+ def use(self, *components) -> Agent:
329
+ for component in components:
330
+ component.setup(self)
331
+ return self
332
+
333
+ def register_tool(self, tooldef: ToolDef) -> None:
334
+ self.tools.append(tooldef)
335
+
336
+ def add_system(self, text: str) -> None:
337
+ self.system = f"{self.system}\n\n{text}" if self.system else text
338
+
339
+ def add_post_hook(self, hook: Callable[..., str | None]) -> None:
340
+ self._post_hooks.append(hook)
341
+
342
+ def add_batch_scope(self, scope: Callable[[], AbstractContextManager]) -> None:
343
+ """Register a context-manager factory entered around each tool-call batch.
344
+
345
+ The model emits a whole batch of tool calls from one view of the world, and
346
+ this loop then dispatches them one after another — so a write early in the
347
+ batch is visible to a read later in the same batch, which the model has no
348
+ way to anticipate. A component whose state must not move under its own feet
349
+ mid-turn registers the boundary here; only the component knows what "not
350
+ moving" means for it, and only the loop knows where the turn ends.
351
+ """
352
+ self._batch_scopes.append(scope)
353
+
354
+ def run(self, prompt: str) -> AgentResult:
355
+ messages: list[Message] = []
356
+ if self.system:
357
+ messages.append(Message(role="system", content=self.system))
358
+ messages.append(Message(role="user", content=prompt))
359
+ usage = Usage()
360
+ last_content = ""
361
+ observations_truncated = 0
362
+ observation_bytes_dropped = 0
363
+
364
+ for _ in range(self.max_turns):
365
+ if self.budget is not None and not self.budget.allow("required"):
366
+ # Past the hard ceiling. Stop iterating and hand back what the run
367
+ # already produced: a truncated answer is still scorable, and an
368
+ # exception here would convert a scorable answer into a loss.
369
+ return AgentResult(
370
+ output=last_content,
371
+ messages=messages,
372
+ usage=usage,
373
+ observations_truncated=observations_truncated,
374
+ observation_bytes_dropped=observation_bytes_dropped,
375
+ )
376
+ resp = self._chat(messages)
377
+ usage = usage + resp.usage
378
+ messages.append(resp.message)
379
+ if resp.message.content:
380
+ last_content = resp.message.content
381
+
382
+ if resp.message.tool_calls:
383
+ with ExitStack() as stack:
384
+ for scope in self._batch_scopes:
385
+ stack.enter_context(scope())
386
+ for tc in resp.message.tool_calls:
387
+ observation, dropped = truncate_counted(
388
+ self._dispatch(tc), self.observation_budget
389
+ )
390
+ if dropped:
391
+ observations_truncated += 1
392
+ observation_bytes_dropped += dropped
393
+ messages.append(
394
+ Message(role="tool", content=observation, tool_call_id=tc.id)
395
+ )
396
+ continue
397
+
398
+ output = resp.message.content or ""
399
+ try:
400
+ feedback = self._first_feedback(prompt, output, messages)
401
+ except BantamError as e:
402
+ # Recording only. The exception propagates unchanged — same type,
403
+ # same message, same traceback — so no verdict can move.
404
+ _attach_transcript(e, messages)
405
+ raise
406
+ if feedback is None:
407
+ return AgentResult(
408
+ output=output,
409
+ messages=messages,
410
+ usage=usage,
411
+ observations_truncated=observations_truncated,
412
+ observation_bytes_dropped=observation_bytes_dropped,
413
+ )
414
+ messages.append(Message(role="user", content=feedback))
415
+
416
+ raise MaxTurnsExceeded(f"no final answer within {self.max_turns} turns", messages)
417
+
418
+ def _chat(self, messages: list[Message]):
419
+ """One model call, with the constrained-decoding tier when it is available.
420
+
421
+ Re-checked per turn, as `structured()` does: a 400 on turn 1 flips the client's
422
+ memo and silently drops the tier from there on.
423
+ """
424
+ tools = [t.tool for t in self.tools] or None
425
+ if self.response_format is not None and _supports_response_format(self.client):
426
+ return self.client.chat(messages, tools=tools, response_format=self.response_format)
427
+ return self.client.chat(messages, tools=tools)
428
+
429
+ def _dispatch(self, tc) -> str:
430
+ tooldef = next((t for t in self.tools if t.tool.name == tc.name), None)
431
+ if tooldef is None:
432
+ names = [t.tool.name for t in self.tools]
433
+ return f"error: unknown tool '{tc.name}'. available tools: {names}"
434
+ try:
435
+ parameters = tooldef.tool.parameters
436
+ arguments = select_declared_arguments(
437
+ coerce_arguments(tc.arguments, parameters), parameters
438
+ )
439
+ mistyped = mistyped_arguments(arguments, parameters)
440
+ if mistyped:
441
+ return tool_argument_types(tc.name, mistyped)
442
+ if not handler_accepts(tooldef.handler, arguments):
443
+ return tool_arguments(tc.name, declared_arguments(parameters))
444
+ return str(tooldef.handler(**arguments))
445
+ except Exception as e:
446
+ return tool_failed(tc.name, e)
447
+
448
+ def _first_feedback(
449
+ self, task: str, output: str, messages: list[Message]
450
+ ) -> str | None:
451
+ for hook in self._post_hooks:
452
+ if getattr(hook, "wants_transcript", False):
453
+ feedback = hook(task, output, messages)
454
+ else:
455
+ feedback = hook(task, output)
456
+ if feedback is not None:
457
+ return feedback
458
+ return None
@@ -0,0 +1,90 @@
1
+ name: default
2
+ schema_instruction: "Return ONLY a JSON object matching this JSON Schema. No prose.\n"
3
+ schema_retry: "{error}\nReturn ONLY a JSON object matching the schema."
4
+ critique_feedback: "A reviewer scored your answer {score}/10 (needs >= {threshold}). Feedback: {feedback}\nIf the feedback says a value is missing, unverified, or absent from your evidence, that is a fact you never looked up: call your tools and read the source that has it. Do not reword, generalise, or hedge the previous answer to work around the gap. Then answer again."
5
+ parse_error: "output was not parseable JSON: {detail}"
6
+ validation_error: "JSON does not match schema at '{where}': {detail}"
7
+ json_answer_retry: "Your answer contains no JSON. Restate your final answer as ONLY the JSON requested by the task, with no prose around it."
8
+ loop_note: "(you have now received this exact result {count} times; it will not change. Do something different or give your final answer now)"
9
+ loop_warn: "(STOP calling tools. Give your final answer now, in exactly the format the task asked for.)"
10
+ evidence_line: "{name}({arguments}) -> {observation}"
11
+ evidence_no_observation: "(no observation)"
12
+ evidence_empty: "(no tool calls were made)"
13
+ # The document reader pair's observations (`document_list`, `document_read`). Every string a
14
+ # reader hands back to the model lives here and not in the harness, for the reason the layer
15
+ # rule gives: the mechanics measured in `docread.py` transfer across models, this wording is
16
+ # exactly what does not.
17
+ document_manifest_empty: "no documents are attached to this task"
18
+ document_manifest_part: "{document} ({kind}) part {index} \"{part}\": {rows} rows, numbered 0 to {last}"
19
+ document_manifest_header_row: " row 0 is the header: {row}"
20
+ document_manifest_first_row: " row 1 is the first data row: {row}"
21
+ document_manifest_last_row: " row {index} is the last data row: {row}"
22
+ # What the rendering left out, stated in the one place that says what exists. Measured need
23
+ # (J25, the user's real ~/Downloads, 2026-08-20): `step test.xlsx` is 18.62 MB and the manifest
24
+ # said `4 parts, 28 rows` — all 28 of those rows are empty lines and the file's actual content
25
+ # is 56 embedded PNGs. Every line here is a COUNT: an adjective ("some images") tells a model
26
+ # nothing it can decide with, and silence tells it something false.
27
+ document_manifest_package_media: "{document}: the file also holds {count} embedded file(s) ({what}) totalling {bytes} bytes, which no row can carry — this reader renders no image or embedded object"
28
+ document_manifest_omitted_media: " NOT in those rows: {count} embedded file(s), {bytes} bytes, anchored to this part"
29
+ document_manifest_omitted_blank: " {count} of those {rows} rows carry no cell value at all and render as an empty line"
30
+ document_manifest_omitted_format: " column(s) {where}: {count} cell(s) store a NUMBER under the date/time format {what} — this reader renders the stored serial number verbatim and does not convert it to a date; convert it with that format code if you need one"
31
+ # J25-D3, the PDF reader. Two silences a page-addressable reader can produce that a
32
+ # spreadsheet cannot: a page that rendered no row (a scan, or glyphs no font maps), and
33
+ # characters dropped because the file states no mapping for them. An empty page and a page
34
+ # nobody could read are different facts and the model must be able to tell them apart.
35
+ document_manifest_omitted_unread_page: " this part rendered NO row: {why}. It draws {count} image(s), {bytes} bytes, that no row can carry — no row means this reader recovered no text from the page, which is NOT the same as the page being empty, and there is no OCR here"
36
+ document_manifest_unread_no_operator: "the page ran no text-showing operator at all, so nothing on it was ever text"
37
+ document_manifest_unread_unmapped: "the page ran {show_ops} text-showing operator(s) and every one of the {unmapped} character(s) they showed came through a font that declares no character map, so this reader dropped them rather than guess at them"
38
+ document_manifest_unread_whitespace: "the page ran {show_ops} text-showing operator(s) and this reader mapped every one of the {vouched} character(s) they showed, and every one of those characters is WHITESPACE — the text was recovered and it carries nothing, which is a fact about the page and not a failure of this reader"
39
+ document_manifest_unread_no_character: "the page ran {show_ops} text-showing operator(s) that put no character on the page at all, neither one this reader could map nor one it had to drop"
40
+ document_manifest_omitted_unmapped: " {count} character(s) shown on this part are NOT in those rows: their codes came from font(s) with no /ToUnicode map ({what}), so the file stores indices into a font's own glyphs and says nowhere which character each glyph draws — this reader drops them rather than guess, because a guess here is indistinguishable from content"
41
+ document_manifest_omitted_other: " NOT in those rows: {count} {subject} ({what})"
42
+ document_page_header: "{document} \"{part}\" rows {start}-{end} of {rows}; each line below begins with its own row number"
43
+ document_page_next: "more rows follow: call document_read again with offset={next_offset}"
44
+ document_page_end: "that was the last row of \"{part}\""
45
+ document_page_truncated: "row {index} was too long for one page and was cut: {dropped} bytes dropped"
46
+ document_unknown: "error: no document named {name}; this task has: {available}"
47
+ document_offset_past_end: "error: offset {offset} is past the end of \"{part}\", which has {rows} rows numbered 0 to {last}"
48
+ document_error: "error: {detail}"
49
+ # `bantamkit_read` (the MCP reader, job43) reuses every `document_*` sentence above with the
50
+ # path standing in for `{document}`. Two sentences are genuinely its own: its continuation
51
+ # line names ITS tool, and a part it does not have is a fact about the file, not the task.
52
+ bantamkit_read_page_next: "more rows follow: call bantamkit_read again with offset={next_offset}"
53
+ bantamkit_read_unknown_part: "error: no part named \"{part}\" in {path}; it has: {available}"
54
+ # The `paste` arm's system message (`docs/eval-data/2026-08-20-document-read-bar.md` §10.2,
55
+ # PASTE_MAX_BYTES = 8621 under Amendment 1, 2026-08-20). The arm registers NO tools, so these four lines are the whole of
56
+ # what the model is told ABOUT the corpus; the rows themselves are docread's own rendering,
57
+ # unprefixed and unreformatted, so `paste` and `document_read` differ in DELIVERY and not in
58
+ # content. Clause 4 makes stating the paste's own completeness part of the contract, and
59
+ # `paste_truncated` says the missing rows are unreachable rather than leaving the model to
60
+ # hunt for a tool that is not there: a paste that lets a truncated head read as the whole
61
+ # sheet is a different, worse arm than the one the bar pre-registered.
62
+ document_paste_preamble: "The following document content is attached to this task. Fields in a row are separated by tabs, rows are given in order, and row 0 of each part is its header."
63
+ document_paste_part: "{document} ({kind}) part {index} \"{part}\": {rows} rows, numbered 0 to {last}; {shown} of them are shown below."
64
+ document_paste_complete: " rows 0 to {last_shown} are shown, which is every row of \"{part}\": this copy is COMPLETE."
65
+ document_paste_truncated: " rows 0 to {last_shown} are shown; rows {first_missing} to {last} of \"{part}\" are NOT shown and no tool is attached that can fetch them. This copy is PARTIAL."
66
+ document_paste_none: " no rows of \"{part}\" are shown and no tool is attached that can fetch them."
67
+ # What the model is told when a tool CALL cannot be made or fails, rather than when a tool
68
+ # answers with its own error (`document_unknown` and friends above are that). It lives here
69
+ # for the reason the layer rule gives and for one measured reason: before 2026-08-20 the
70
+ # dispatcher formatted this sentence itself and interpolated the raw exception, so a model
71
+ # that sent one extra argument was handed
72
+ # `_document_tools.<locals>.list_documents() got an unexpected keyword argument 'document'`
73
+ # — a Python qualname, model-facing, owned by no asset and pinned by no golden. The
74
+ # argument-shaped failures are intercepted before the handler is called now, so `{detail}`
75
+ # carries a handler's OWN sentence and no longer a signature fragment.
76
+ tool_failed: "error: {tool} failed: {detail}. fix the arguments and retry."
77
+ tool_arguments: "error: {tool} does not take the arguments it was given. it takes: {accepts}. fix the arguments and retry."
78
+ tool_arguments_none: "error: {tool} takes no arguments at all. call it with none and retry."
79
+ # 2026-08-20, `RB-P86`. The sibling of the two above, one layer in. `select_declared_arguments`
80
+ # drops keys the schema does not declare and type-checks nothing, so `llama3.2:3b` — which
81
+ # emits a JSON-Schema FRAGMENT as the value of a declared parameter,
82
+ # `{"document": {"description": "stock", "type": "string"}}` — got its dict straight through to
83
+ # `docs.get(name)` and the model read `unhashable type: 'dict'`, thirteen times over 432 graded
84
+ # runs. That is CPython's sentence about a hash table, not a tool's sentence about its own
85
+ # arguments, and a 3b that receives it stops calling tools and answers that it cannot access
86
+ # the workbook. What it is told now names the argument, the type the tool declares and the type
87
+ # it was sent, so the next turn has something to correct. One frame for one or many, because
88
+ # the count is not the thing the model has to act on.
89
+ tool_argument_types: "error: {tool} was called with the wrong type of argument. {problems}. fix the arguments and retry."
90
+ tool_argument_type: "{argument} must be type {expected}, not type {actual}"