askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
askback/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""An evaluation framework for conversational agents.
|
|
2
|
+
|
|
3
|
+
Drives an agent through test conversations, grades the replies with layered
|
|
4
|
+
evaluators, and gates on the result. The agent is reached through an adapter
|
|
5
|
+
(`askback.adapters`), so the same suite grades a deployed service over HTTP or
|
|
6
|
+
the `askback_sim` fixture in this process. This package never imports `askback_sim`.
|
|
7
|
+
|
|
8
|
+
Extension points:
|
|
9
|
+
Adapter askback.adapters.base.AgentAdapter
|
|
10
|
+
Evaluator askback.evaluation.base.Evaluator
|
|
11
|
+
LLM provider askback.providers.base.LLMProvider
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from .core.config import AppConfig
|
|
17
|
+
from .core.contracts import ChatRequest, ChatResponse, ResponseStatus
|
|
18
|
+
from .core.logging import setup_logging
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"AppConfig",
|
|
24
|
+
"ChatRequest",
|
|
25
|
+
"ChatResponse",
|
|
26
|
+
"ResponseStatus",
|
|
27
|
+
"__version__",
|
|
28
|
+
"setup_logging",
|
|
29
|
+
]
|
askback/__main__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Adapters: the seam between the harness and the system under test.
|
|
2
|
+
|
|
3
|
+
The runner depends on `AgentAdapter` and nothing more specific, which is what
|
|
4
|
+
lets one suite file grade a simulator and a deployed agent without changing.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from .base import AgentAdapter
|
|
10
|
+
|
|
11
|
+
__all__ = ["AgentAdapter"]
|
askback/adapters/base.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""The adapter seam: how the harness reaches the system under test.
|
|
2
|
+
|
|
3
|
+
Everything upstream of this - suites, the runner, evaluators, gates - should be
|
|
4
|
+
indifferent to whether the agent is a simulator in this process or a deployed
|
|
5
|
+
service behind HTTP. An adapter is the one place that difference lives: it takes
|
|
6
|
+
a user turn and hands back a `ChatResponse`, the vocabulary the evaluators
|
|
7
|
+
already grade.
|
|
8
|
+
|
|
9
|
+
`AgentAdapter` is a Protocol, not a base class. An adapter implements a shape; it
|
|
10
|
+
does not inherit from the framework, so a team can write one around their own
|
|
11
|
+
client without subclassing anything here.
|
|
12
|
+
|
|
13
|
+
To add an adapter:
|
|
14
|
+
|
|
15
|
+
class MyAdapter:
|
|
16
|
+
def __enter__(self) -> MyAdapter:
|
|
17
|
+
self._client = open_client() # once per run, not per call
|
|
18
|
+
return self
|
|
19
|
+
|
|
20
|
+
def __exit__(self, *exc: object) -> None:
|
|
21
|
+
self._client.close()
|
|
22
|
+
|
|
23
|
+
def send(self, message, *, session_id, principal="default") -> ChatResponse:
|
|
24
|
+
payload = self._client.ask(message, session=session_id)
|
|
25
|
+
return ChatResponse(reply=payload["text"], conversation_id=session_id)
|
|
26
|
+
|
|
27
|
+
def describe(self) -> dict[str, str]:
|
|
28
|
+
return {"adapter": "my-adapter", "model": self._model}
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import threading
|
|
34
|
+
from types import TracebackType
|
|
35
|
+
from typing import Protocol, TypeVar, runtime_checkable
|
|
36
|
+
|
|
37
|
+
from ..core.contracts import ChatResponse
|
|
38
|
+
|
|
39
|
+
_T = TypeVar("_T")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@runtime_checkable
|
|
43
|
+
class AgentAdapter(Protocol):
|
|
44
|
+
"""One system under test, as the runner sees it.
|
|
45
|
+
|
|
46
|
+
Synchronous on purpose. The runner is a thread pool and the work is I/O
|
|
47
|
+
bound, so threads already give the concurrency; an async seam would push an
|
|
48
|
+
event loop into every caller and every adapter author for no measured gain.
|
|
49
|
+
|
|
50
|
+
`isinstance(x, AgentAdapter)` only confirms the four methods exist. It says
|
|
51
|
+
nothing about their signatures - that is what the type checker is for.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def __enter__(self) -> AgentAdapter:
|
|
55
|
+
"""Open whatever the adapter needs for a whole run, and return it.
|
|
56
|
+
|
|
57
|
+
The lifecycle belongs to the adapter so that an HTTP implementation can
|
|
58
|
+
hold one pooled client for every case instead of paying a connection per
|
|
59
|
+
call. `send` is only valid between `__enter__` and `__exit__`.
|
|
60
|
+
"""
|
|
61
|
+
...
|
|
62
|
+
|
|
63
|
+
def __exit__(
|
|
64
|
+
self,
|
|
65
|
+
exc_type: type[BaseException] | None,
|
|
66
|
+
exc: BaseException | None,
|
|
67
|
+
traceback: TracebackType | None,
|
|
68
|
+
/,
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Release the transport. Runs whether the run finished or blew up.
|
|
71
|
+
|
|
72
|
+
Returns None rather than bool: an adapter has no business suppressing an
|
|
73
|
+
exception raised by the run it was carrying.
|
|
74
|
+
"""
|
|
75
|
+
...
|
|
76
|
+
|
|
77
|
+
def send(
|
|
78
|
+
self, message: str, *, session_id: str, principal: str = "default"
|
|
79
|
+
) -> ChatResponse:
|
|
80
|
+
"""Deliver one user turn and return what the system said.
|
|
81
|
+
|
|
82
|
+
`session_id` is chosen by the caller. Turns sharing one belong to the
|
|
83
|
+
same conversation and must see each other's history; turns with
|
|
84
|
+
different ids must not, which is what keeps test cases from
|
|
85
|
+
contaminating one another.
|
|
86
|
+
|
|
87
|
+
`principal` names the identity the turn is sent as. The adapter resolves
|
|
88
|
+
the name to whatever the system needs - a token, headers, a user and
|
|
89
|
+
tenant. It is a label, never the credential itself: it is recorded in
|
|
90
|
+
reports.
|
|
91
|
+
|
|
92
|
+
MUST be safe to call concurrently from multiple threads. The runner
|
|
93
|
+
sends different sessions from different workers at the same time, so
|
|
94
|
+
anything shared across calls - a client, a token cache, a session map -
|
|
95
|
+
has to tolerate that. Turns within one session arrive in order, never
|
|
96
|
+
interleaved. Nothing in this module can enforce this; it is a contract
|
|
97
|
+
on whoever implements it, and breaking it shows up as cross-case
|
|
98
|
+
contamination rather than as an exception.
|
|
99
|
+
|
|
100
|
+
Report only what the system returned. A field it does not expose stays
|
|
101
|
+
empty - no retrieved chunks, no citations - so the evaluators that need
|
|
102
|
+
it call the case unmeasurable instead of grading an invention. Raise
|
|
103
|
+
when the turn could not be delivered or its reply could not be read; the
|
|
104
|
+
runner records that as a failed case. Do not synthesise a response to
|
|
105
|
+
stand in for one.
|
|
106
|
+
"""
|
|
107
|
+
...
|
|
108
|
+
|
|
109
|
+
def describe(self) -> dict[str, str]:
|
|
110
|
+
"""Provenance for the run report: which system this was.
|
|
111
|
+
|
|
112
|
+
The runner cannot reach inside an adapter for a model or retriever name
|
|
113
|
+
the way it could with an in-process engine, so the adapter states its
|
|
114
|
+
own. The values land in `RunReport.context`, are written into every
|
|
115
|
+
report, and are what the regression comparison reads to decide whether
|
|
116
|
+
two runs are comparable at all.
|
|
117
|
+
|
|
118
|
+
The keys are the implementer's choice. Include whatever would make two
|
|
119
|
+
runs differ for reasons other than the agent's behaviour - endpoint,
|
|
120
|
+
model, deployed version, corpus revision - and keep the keys stable from
|
|
121
|
+
run to run: a renamed key compares as nothing. Strings only, because the
|
|
122
|
+
mapping is serialised and compared by equality. Never a secret; reports
|
|
123
|
+
are artifacts.
|
|
124
|
+
|
|
125
|
+
Must not depend on a turn having been sent: the runner records it when
|
|
126
|
+
the run starts.
|
|
127
|
+
"""
|
|
128
|
+
...
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class AdapterLifecycle:
|
|
132
|
+
"""Reentrant `with` support shared by adapters, so each does not re-derive it.
|
|
133
|
+
|
|
134
|
+
A depth, not a flag: one adapter can be entered by several runs, and the
|
|
135
|
+
first to finish must not close it under the others. A subclass calls
|
|
136
|
+
`super().__init__()`, then guards `send` and `describe` with
|
|
137
|
+
`_require_open`.
|
|
138
|
+
|
|
139
|
+
An adapter that owns a transport (an HTTP client) overrides `_open` and
|
|
140
|
+
`_close`. They run under the counter's lock on the first entry and the
|
|
141
|
+
last exit, so the transport is built once per run, never per `with` or per
|
|
142
|
+
call, and a racing second entry cannot see it half-built. If `_open`
|
|
143
|
+
raises, the adapter stays closed.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
def __init__(self) -> None:
|
|
147
|
+
self._depth = 0
|
|
148
|
+
# Guards the counter only. `send` is never serialised behind it.
|
|
149
|
+
self._depth_lock = threading.Lock()
|
|
150
|
+
|
|
151
|
+
def __enter__(self: _T) -> _T:
|
|
152
|
+
assert isinstance(self, AdapterLifecycle)
|
|
153
|
+
with self._depth_lock:
|
|
154
|
+
if self._depth == 0:
|
|
155
|
+
self._open()
|
|
156
|
+
self._depth += 1
|
|
157
|
+
return self
|
|
158
|
+
|
|
159
|
+
def __exit__(
|
|
160
|
+
self,
|
|
161
|
+
exc_type: type[BaseException] | None,
|
|
162
|
+
exc: BaseException | None,
|
|
163
|
+
traceback: TracebackType | None,
|
|
164
|
+
/,
|
|
165
|
+
) -> None:
|
|
166
|
+
with self._depth_lock:
|
|
167
|
+
# Floored so a stray manual __exit__ cannot leave the next `with`
|
|
168
|
+
# block closed from the inside.
|
|
169
|
+
if self._depth == 0:
|
|
170
|
+
return
|
|
171
|
+
self._depth -= 1
|
|
172
|
+
if self._depth == 0:
|
|
173
|
+
self._close()
|
|
174
|
+
|
|
175
|
+
def _open(self) -> None:
|
|
176
|
+
"""Acquire the transport. Default: there is none."""
|
|
177
|
+
|
|
178
|
+
def _close(self) -> None:
|
|
179
|
+
"""Release the transport. Must tolerate being called after a failed run."""
|
|
180
|
+
|
|
181
|
+
def _require_open(self, method: str) -> None:
|
|
182
|
+
if self._depth == 0:
|
|
183
|
+
raise RuntimeError(
|
|
184
|
+
f"{type(self).__name__}.{method}() called outside its `with` block; "
|
|
185
|
+
"enter the adapter before using it."
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
__all__ = ["AdapterLifecycle", "AgentAdapter"]
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""Choosing the adapter from config.
|
|
2
|
+
|
|
3
|
+
`agent.adapter` in your config file names the system under test, so pointing a run
|
|
4
|
+
at something else is a config edit, not a code change. Callers get an
|
|
5
|
+
`AgentAdapter` back and never learn which one it is.
|
|
6
|
+
|
|
7
|
+
This module never imports the simulator, not even lazily. `in_process` needs an
|
|
8
|
+
engine, and the caller - the CLI or the API, which are allowed to know the
|
|
9
|
+
simulator exists - builds it and passes it in. A factory that reached for the
|
|
10
|
+
simulator itself would be the one exception the package boundary could not
|
|
11
|
+
survive having.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ..core.config import AppConfig
|
|
20
|
+
from ..core.errors import ConfigError
|
|
21
|
+
from ..evaluation.safety_eval import declare_side_effecting_tools
|
|
22
|
+
from .base import AgentAdapter
|
|
23
|
+
from .in_process import DEFAULT_PRINCIPALS, InProcessAdapter, InProcessEngine
|
|
24
|
+
|
|
25
|
+
#: What a config file with no `agent:` section runs against. Configs written before
|
|
26
|
+
#: the adapter seam existed have none, and they always meant the simulator. A run with no
|
|
27
|
+
#: config file at all never gets here: it is the built-in demo (see `askback.demo`).
|
|
28
|
+
DEFAULT_ADAPTER = "in_process"
|
|
29
|
+
|
|
30
|
+
VALID_ADAPTERS = ("in_process", "http", "socketio", "demo")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def adapter_name(settings: AppConfig) -> str:
|
|
34
|
+
"""Which adapter `agent.adapter` names, so a caller knows whether to build an engine."""
|
|
35
|
+
return str(_agent_section(settings).get("adapter", DEFAULT_ADAPTER))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def build_adapter(
|
|
39
|
+
settings: AppConfig, *, engine: InProcessEngine | None = None
|
|
40
|
+
) -> AgentAdapter:
|
|
41
|
+
"""Build the adapter named by `agent.adapter`.
|
|
42
|
+
|
|
43
|
+
`engine` is the in-process system under test, and is required when the
|
|
44
|
+
adapter is `in_process`: this factory does not build one, because that
|
|
45
|
+
would mean importing the simulator. It is ignored by every other adapter.
|
|
46
|
+
|
|
47
|
+
`agent.side_effecting_tools`, when present, is registered here as a
|
|
48
|
+
declaration source for the tool-misuse check. Here rather than in each
|
|
49
|
+
caller because this is the one place every run names its system under
|
|
50
|
+
test, and that system is the only thing that can know its tools.
|
|
51
|
+
"""
|
|
52
|
+
agent = _agent_section(settings)
|
|
53
|
+
|
|
54
|
+
tools = _side_effecting_tools(agent)
|
|
55
|
+
if tools is not None:
|
|
56
|
+
declare_side_effecting_tools(lambda: tools)
|
|
57
|
+
|
|
58
|
+
name = agent.get("adapter", DEFAULT_ADAPTER)
|
|
59
|
+
if name == "in_process":
|
|
60
|
+
if engine is None:
|
|
61
|
+
raise ConfigError(
|
|
62
|
+
"agent.adapter 'in_process' drives an engine in this process, and none "
|
|
63
|
+
"was passed to build_adapter(engine=...). The example simulator provides one "
|
|
64
|
+
"from a source checkout: see examples/hr."
|
|
65
|
+
)
|
|
66
|
+
return InProcessAdapter(engine, _principals(agent))
|
|
67
|
+
|
|
68
|
+
if name == "http":
|
|
69
|
+
try:
|
|
70
|
+
from .http import HTTPAdapter
|
|
71
|
+
except ImportError as exc:
|
|
72
|
+
raise ConfigError(
|
|
73
|
+
"agent.adapter 'http' needs httpx. Run: pip install -e .[http]"
|
|
74
|
+
) from exc
|
|
75
|
+
|
|
76
|
+
return HTTPAdapter.from_config(agent)
|
|
77
|
+
|
|
78
|
+
if name == "demo":
|
|
79
|
+
from ..demo import DemoAdapter
|
|
80
|
+
|
|
81
|
+
return DemoAdapter.from_config(agent)
|
|
82
|
+
|
|
83
|
+
if name == "socketio":
|
|
84
|
+
# No ImportError guard here: that module imports python-socketio when it
|
|
85
|
+
# builds, and turns its absence into a ConfigError carrying the install hint.
|
|
86
|
+
from .socketio_adapter import SocketIOAdapter
|
|
87
|
+
|
|
88
|
+
return SocketIOAdapter.from_config(agent)
|
|
89
|
+
|
|
90
|
+
raise ConfigError(
|
|
91
|
+
f"unknown agent.adapter {name!r}; valid options: {', '.join(VALID_ADAPTERS)}"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def resolve_principals(settings: AppConfig) -> dict[str, tuple[str, str]]:
|
|
96
|
+
"""The principal mapping a run will use, for checking datasets before it starts.
|
|
97
|
+
|
|
98
|
+
The configured one, or the adapter's defaults when `agent.principals` is
|
|
99
|
+
absent. Only the in-process adapter resolves identities to tenants; any
|
|
100
|
+
other adapter yields an empty mapping, which makes the loader check a no-op.
|
|
101
|
+
"""
|
|
102
|
+
agent = _agent_section(settings)
|
|
103
|
+
if agent.get("adapter", DEFAULT_ADAPTER) != "in_process":
|
|
104
|
+
# An HTTP identity is a header, not a (user_id, tenant_id) pair, so
|
|
105
|
+
# there is no tenant to check a dataset's declaration against.
|
|
106
|
+
return {}
|
|
107
|
+
configured = _principals(agent)
|
|
108
|
+
if configured is not None:
|
|
109
|
+
return configured
|
|
110
|
+
return dict(DEFAULT_PRINCIPALS)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _agent_section(settings: AppConfig) -> Mapping[str, Any]:
|
|
114
|
+
agent = settings.raw.get("agent", {})
|
|
115
|
+
if not isinstance(agent, Mapping):
|
|
116
|
+
raise ConfigError(f"config `agent` must be a mapping, got {agent!r}")
|
|
117
|
+
return agent
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _principals(agent: Mapping[str, Any]) -> dict[str, tuple[str, str]] | None:
|
|
121
|
+
"""Read `agent.principals` into the (user_id, tenant_id) pairs an adapter takes.
|
|
122
|
+
|
|
123
|
+
None when the key is absent, which leaves the adapter on its own defaults.
|
|
124
|
+
Anything present is checked here so a malformed entry is reported with its
|
|
125
|
+
config path, before a case is sent as it.
|
|
126
|
+
"""
|
|
127
|
+
declared = agent.get("principals")
|
|
128
|
+
if declared is None:
|
|
129
|
+
return None
|
|
130
|
+
if not isinstance(declared, Mapping):
|
|
131
|
+
raise ConfigError(
|
|
132
|
+
"agent.principals must map a principal label to "
|
|
133
|
+
f"{{user_id, tenant_id}}, got {declared!r}"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
principals: dict[str, tuple[str, str]] = {}
|
|
137
|
+
for label, identity in declared.items():
|
|
138
|
+
if not (
|
|
139
|
+
isinstance(label, str)
|
|
140
|
+
and isinstance(identity, Mapping)
|
|
141
|
+
and set(identity) == {"user_id", "tenant_id"}
|
|
142
|
+
and all(isinstance(part, str) for part in identity.values())
|
|
143
|
+
):
|
|
144
|
+
raise ConfigError(
|
|
145
|
+
f"agent.principals.{label} must be a mapping with exactly a string "
|
|
146
|
+
f"user_id and a string tenant_id, got {identity!r}"
|
|
147
|
+
)
|
|
148
|
+
principals[label] = (identity["user_id"], identity["tenant_id"])
|
|
149
|
+
return principals
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _side_effecting_tools(agent: Mapping[str, Any]) -> frozenset[str] | None:
|
|
153
|
+
"""Read `agent.side_effecting_tools`: the tools this agent says act on the world.
|
|
154
|
+
|
|
155
|
+
None when the key is absent, and absent is not the same as empty: `[]`
|
|
156
|
+
states that no tool has side effects, and the check can then pass. Absent
|
|
157
|
+
states nothing, so the check keeps failing until someone does (invariant 8).
|
|
158
|
+
A bare string is refused; iterating it would declare its letters.
|
|
159
|
+
"""
|
|
160
|
+
declared = agent.get("side_effecting_tools")
|
|
161
|
+
if declared is None:
|
|
162
|
+
return None
|
|
163
|
+
if not isinstance(declared, list) or not all(isinstance(name, str) for name in declared):
|
|
164
|
+
raise ConfigError(
|
|
165
|
+
f"agent.side_effecting_tools must be a list of tool names, got {declared!r}"
|
|
166
|
+
)
|
|
167
|
+
return frozenset(declared)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
__all__ = [
|
|
171
|
+
"DEFAULT_ADAPTER",
|
|
172
|
+
"VALID_ADAPTERS",
|
|
173
|
+
"adapter_name",
|
|
174
|
+
"build_adapter",
|
|
175
|
+
"resolve_principals",
|
|
176
|
+
]
|