askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
askback/adapters/http.py
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
"""The HTTP adapter: a deployed agent, reached over the wire.
|
|
2
|
+
|
|
3
|
+
This is the adapter that makes the harness a tool rather than a demo. The agent's
|
|
4
|
+
payload is its own; this module owns the three things that differ between
|
|
5
|
+
agents - how to ask (a request template), who to ask as (per-principal headers,
|
|
6
|
+
body fields, or both), and where the answer lives (dotted response paths) - so a
|
|
7
|
+
payload change is a YAML edit, not a code change.
|
|
8
|
+
|
|
9
|
+
Needs httpx, from the `http` extra. Nothing imports this module until
|
|
10
|
+
`agent.adapter: http` asks for it, so the core stays importable without it.
|
|
11
|
+
|
|
12
|
+
Retries, and what they cost. A turn is retried on transport errors and on
|
|
13
|
+
408/409/425/429/500/502/503/504, up to `max_attempts` tries in all. Any other
|
|
14
|
+
4xx is a contract break - a 422 means our request is wrong, and sending it again
|
|
15
|
+
only burns wall-clock and hides the cause - so it fails at once. A retried POST
|
|
16
|
+
may deliver a turn TWICE: when a response is lost after the agent acted, the
|
|
17
|
+
agent sees the same message again and may record it twice in its history. For an
|
|
18
|
+
agent that is not idempotent per `session_id`, set `max_attempts: 1` and accept
|
|
19
|
+
transient failures as failed cases.
|
|
20
|
+
|
|
21
|
+
Honest reporting. A field the agent does not return stays empty - no citations,
|
|
22
|
+
no retrieved chunks, no tool calls - so the evaluators that need it call the case
|
|
23
|
+
unmeasurable. A status that is absent or not one we know becomes
|
|
24
|
+
`ResponseStatus.UNKNOWN`, never `ANSWERED`: defaulting to success is how a
|
|
25
|
+
broken integration goes green.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import copy
|
|
31
|
+
import email.utils
|
|
32
|
+
import random
|
|
33
|
+
import time
|
|
34
|
+
from collections.abc import Callable, Mapping
|
|
35
|
+
from datetime import UTC, datetime
|
|
36
|
+
from typing import Any
|
|
37
|
+
from urllib.parse import urlsplit
|
|
38
|
+
|
|
39
|
+
import httpx
|
|
40
|
+
|
|
41
|
+
from ..core.contracts import (
|
|
42
|
+
ChatResponse,
|
|
43
|
+
Intent,
|
|
44
|
+
)
|
|
45
|
+
from ..core.errors import AgentCallError, ConfigError
|
|
46
|
+
from ..core.logging import get_logger
|
|
47
|
+
from .base import AdapterLifecycle
|
|
48
|
+
from .payload import DEFAULT_RESPONSE_MAPPING
|
|
49
|
+
from .payload import MISSING as _MISSING
|
|
50
|
+
from .payload import chunk as _chunk
|
|
51
|
+
from .payload import citation as _citation
|
|
52
|
+
from .payload import dig as _dig
|
|
53
|
+
from .payload import fill as _fill
|
|
54
|
+
from .payload import status_from as _status
|
|
55
|
+
from .payload import tool as _tool
|
|
56
|
+
from .payload import validated_headers as _validated_headers
|
|
57
|
+
from .payload import validated_mapping as _validated_mapping
|
|
58
|
+
from .payload import validated_template as _validated_template
|
|
59
|
+
|
|
60
|
+
log = get_logger(__name__)
|
|
61
|
+
|
|
62
|
+
#: Retried alongside any `httpx.TransportError`. Everything else non-2xx fails.
|
|
63
|
+
RETRYABLE_STATUSES = frozenset({408, 409, 425, 429, 500, 502, 503, 504})
|
|
64
|
+
|
|
65
|
+
#: Longest a `Retry-After` is honoured for. An agent asking for an hour would
|
|
66
|
+
#: otherwise park a worker, and with it the run, for an hour.
|
|
67
|
+
MAX_RETRY_AFTER_S = 60.0
|
|
68
|
+
|
|
69
|
+
#: Longest the computed (not server-requested) backoff grows to.
|
|
70
|
+
MAX_BACKOFF_S = 30.0
|
|
71
|
+
|
|
72
|
+
DEFAULT_REQUEST_TEMPLATE: Mapping[str, Any] = {
|
|
73
|
+
"query": "{message}",
|
|
74
|
+
"session_id": "{session_id}",
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class HTTPAdapter(AdapterLifecycle):
|
|
80
|
+
"""`AgentAdapter` over one HTTP endpoint that takes a turn and returns JSON."""
|
|
81
|
+
|
|
82
|
+
def __init__(
|
|
83
|
+
self,
|
|
84
|
+
url: str,
|
|
85
|
+
*,
|
|
86
|
+
principal_headers: Mapping[str, Mapping[str, str]],
|
|
87
|
+
principal_fields: Mapping[str, Mapping[str, Any]] | None = None,
|
|
88
|
+
request_template: Mapping[str, Any] | None = None,
|
|
89
|
+
response_mapping: Mapping[str, str] | None = None,
|
|
90
|
+
headers: Mapping[str, str] | None = None,
|
|
91
|
+
timeout_s: float = 30.0,
|
|
92
|
+
max_attempts: int = 3,
|
|
93
|
+
backoff_s: float = 0.5,
|
|
94
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
95
|
+
transport: httpx.BaseTransport | None = None,
|
|
96
|
+
) -> None:
|
|
97
|
+
super().__init__()
|
|
98
|
+
parts = urlsplit(url)
|
|
99
|
+
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
100
|
+
raise ConfigError(f"agent.url must be an http(s) URL, got {url!r}")
|
|
101
|
+
if timeout_s <= 0:
|
|
102
|
+
raise ConfigError(f"agent.timeout_s must be positive, got {timeout_s!r}")
|
|
103
|
+
if max_attempts < 1:
|
|
104
|
+
raise ConfigError(f"agent.max_attempts must be at least 1, got {max_attempts!r}")
|
|
105
|
+
if backoff_s < 0:
|
|
106
|
+
raise ConfigError(f"agent.backoff_s cannot be negative, got {backoff_s!r}")
|
|
107
|
+
|
|
108
|
+
self._url = url
|
|
109
|
+
# Credentials and query strings never reach a report: it is an artifact.
|
|
110
|
+
self._endpoint = f"{parts.scheme}://{parts.hostname}{f':{parts.port}' if parts.port else ''}{parts.path}"
|
|
111
|
+
self._template = _validated_template(request_template or DEFAULT_REQUEST_TEMPLATE)
|
|
112
|
+
self._mapping = _validated_mapping(response_mapping)
|
|
113
|
+
self._headers = _validated_headers(headers or {}, "agent.headers")
|
|
114
|
+
# Copied, validated, and deep enough that a caller mutating its own
|
|
115
|
+
# dicts mid-run cannot change who a later turn is sent as.
|
|
116
|
+
self._principals = {
|
|
117
|
+
label: _validated_headers(value, f"agent.principal_headers.{label}")
|
|
118
|
+
for label, value in principal_headers.items()
|
|
119
|
+
}
|
|
120
|
+
self._fields = _validated_fields(principal_fields, self._principals, self._template)
|
|
121
|
+
self._timeout_s = timeout_s
|
|
122
|
+
self._max_attempts = max_attempts
|
|
123
|
+
self._backoff_s = backoff_s
|
|
124
|
+
self._sleep = sleep
|
|
125
|
+
# httpx's own seam for where requests go: an ASGI app, a proxy, a mock.
|
|
126
|
+
self._transport = transport
|
|
127
|
+
self._client: httpx.Client | None = None
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def from_config(cls, agent: Mapping[str, Any]) -> HTTPAdapter:
|
|
131
|
+
"""Build from the `agent:` section of your config file."""
|
|
132
|
+
known = {
|
|
133
|
+
"adapter", "url", "timeout_s", "max_attempts", "backoff_s", "headers",
|
|
134
|
+
"request_template", "response_mapping", "principal_headers", "principal_fields",
|
|
135
|
+
# Read by the factory, which registers it before this runs; named here
|
|
136
|
+
# only so the stray-key check does not refuse the one key an HTTP run
|
|
137
|
+
# needs to make `tool_side_effects` measurable.
|
|
138
|
+
"side_effecting_tools",
|
|
139
|
+
} # fmt: skip
|
|
140
|
+
stray = sorted(set(agent) - known)
|
|
141
|
+
if stray:
|
|
142
|
+
raise ConfigError(
|
|
143
|
+
f"unknown agent key(s) for the http adapter: {stray}; known: {sorted(known)}"
|
|
144
|
+
)
|
|
145
|
+
url = agent.get("url")
|
|
146
|
+
if not isinstance(url, str) or not url:
|
|
147
|
+
raise ConfigError("agent.url is required for the http adapter")
|
|
148
|
+
principals = agent.get("principal_headers")
|
|
149
|
+
if not isinstance(principals, Mapping):
|
|
150
|
+
raise ConfigError(
|
|
151
|
+
"agent.principal_headers is required for the http adapter: a mapping of "
|
|
152
|
+
"principal label -> {Header: value}. Use `default: {}` for an agent with "
|
|
153
|
+
"no authentication; a missing label never falls back to anonymous."
|
|
154
|
+
)
|
|
155
|
+
for label, value in principals.items():
|
|
156
|
+
if not isinstance(value, Mapping):
|
|
157
|
+
raise ConfigError(
|
|
158
|
+
f"agent.principal_headers.{label} must be a mapping, got {value!r}"
|
|
159
|
+
)
|
|
160
|
+
return cls(
|
|
161
|
+
url,
|
|
162
|
+
principal_headers=principals,
|
|
163
|
+
principal_fields=_optional_mapping(agent, "principal_fields"),
|
|
164
|
+
request_template=_optional_mapping(agent, "request_template"),
|
|
165
|
+
response_mapping=_optional_mapping(agent, "response_mapping"),
|
|
166
|
+
headers=_optional_mapping(agent, "headers"),
|
|
167
|
+
timeout_s=_number(agent, "timeout_s", 30.0),
|
|
168
|
+
max_attempts=int(_number(agent, "max_attempts", 3)),
|
|
169
|
+
backoff_s=_number(agent, "backoff_s", 0.5),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
# ------------------------------------------------------------------ #
|
|
173
|
+
# Lifecycle: one pooled client for the whole run, never one per call.
|
|
174
|
+
# ------------------------------------------------------------------ #
|
|
175
|
+
|
|
176
|
+
def _open(self) -> None:
|
|
177
|
+
self._client = httpx.Client(
|
|
178
|
+
timeout=self._timeout_s, headers=self._headers, transport=self._transport
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def _close(self) -> None:
|
|
182
|
+
client, self._client = self._client, None
|
|
183
|
+
if client is not None:
|
|
184
|
+
client.close()
|
|
185
|
+
|
|
186
|
+
# ------------------------------------------------------------------ #
|
|
187
|
+
# AgentAdapter
|
|
188
|
+
# ------------------------------------------------------------------ #
|
|
189
|
+
|
|
190
|
+
def send(
|
|
191
|
+
self, message: str, *, session_id: str, principal: str = "default"
|
|
192
|
+
) -> ChatResponse:
|
|
193
|
+
"""Deliver one turn, retrying only what is worth retrying, and map the reply.
|
|
194
|
+
|
|
195
|
+
No lock: `httpx.Client` is safe for concurrent use, and nothing else
|
|
196
|
+
here is shared and mutable.
|
|
197
|
+
"""
|
|
198
|
+
self._require_open("send")
|
|
199
|
+
client = self._client
|
|
200
|
+
if client is None: # unreachable while the depth guard holds; keeps mypy honest
|
|
201
|
+
raise RuntimeError("HTTPAdapter has no open client")
|
|
202
|
+
headers = self._principal_headers(principal)
|
|
203
|
+
body = _fill(self._template, {"message": message, "session_id": session_id})
|
|
204
|
+
# Disjoint from the template's keys by construction, so this only adds.
|
|
205
|
+
body.update(copy.deepcopy(self._fields.get(principal, {})))
|
|
206
|
+
|
|
207
|
+
response, latency_ms = self._post_with_retry(client, body, headers, session_id)
|
|
208
|
+
return self._map(_json_object(response), session_id, latency_ms)
|
|
209
|
+
|
|
210
|
+
def describe(self) -> dict[str, str]:
|
|
211
|
+
"""Provenance for the run report.
|
|
212
|
+
|
|
213
|
+
The endpoint without credentials or query string. Header values are never
|
|
214
|
+
recorded: they are tokens, and a report is an artifact.
|
|
215
|
+
"""
|
|
216
|
+
self._require_open("describe")
|
|
217
|
+
return {"adapter": "http", "endpoint": self._endpoint}
|
|
218
|
+
|
|
219
|
+
# ------------------------------------------------------------------ #
|
|
220
|
+
# Transport
|
|
221
|
+
# ------------------------------------------------------------------ #
|
|
222
|
+
|
|
223
|
+
def _principal_headers(self, principal: str) -> dict[str, str]:
|
|
224
|
+
"""Resolve a label to headers, or refuse. Never falls back to anonymous.
|
|
225
|
+
|
|
226
|
+
An isolation case that quietly ran with no credentials, or someone
|
|
227
|
+
else's, would pass for the wrong reason.
|
|
228
|
+
"""
|
|
229
|
+
try:
|
|
230
|
+
return self._principals[principal]
|
|
231
|
+
except KeyError:
|
|
232
|
+
raise ConfigError(
|
|
233
|
+
f"unknown principal {principal!r}; agent.principal_headers knows: "
|
|
234
|
+
f"{sorted(self._principals)}"
|
|
235
|
+
) from None
|
|
236
|
+
|
|
237
|
+
def _post_with_retry(
|
|
238
|
+
self,
|
|
239
|
+
client: httpx.Client,
|
|
240
|
+
body: Any,
|
|
241
|
+
headers: dict[str, str],
|
|
242
|
+
session_id: str,
|
|
243
|
+
) -> tuple[httpx.Response, float]:
|
|
244
|
+
for attempt in range(1, self._max_attempts + 1):
|
|
245
|
+
last = attempt == self._max_attempts
|
|
246
|
+
started = time.perf_counter()
|
|
247
|
+
try:
|
|
248
|
+
response = client.post(self._url, json=body, headers=headers)
|
|
249
|
+
except httpx.TransportError as exc:
|
|
250
|
+
if last:
|
|
251
|
+
raise AgentCallError(
|
|
252
|
+
f"{self._endpoint} unreachable after {attempt} attempt(s) "
|
|
253
|
+
f"for session {session_id!r}: {type(exc).__name__}: {exc}"
|
|
254
|
+
) from exc
|
|
255
|
+
self._wait(attempt, None, f"{type(exc).__name__}")
|
|
256
|
+
continue
|
|
257
|
+
latency_ms = (time.perf_counter() - started) * 1000
|
|
258
|
+
|
|
259
|
+
if response.is_success:
|
|
260
|
+
return response, latency_ms
|
|
261
|
+
if response.status_code not in RETRYABLE_STATUSES:
|
|
262
|
+
raise AgentCallError(
|
|
263
|
+
f"{self._endpoint} answered HTTP {response.status_code}, which is not "
|
|
264
|
+
f"retried: {_snippet(response)}"
|
|
265
|
+
)
|
|
266
|
+
if last:
|
|
267
|
+
raise AgentCallError(
|
|
268
|
+
f"{self._endpoint} still answering HTTP {response.status_code} after "
|
|
269
|
+
f"{attempt} attempt(s): {_snippet(response)}"
|
|
270
|
+
)
|
|
271
|
+
self._wait(
|
|
272
|
+
attempt,
|
|
273
|
+
_retry_after(response.headers.get("Retry-After")),
|
|
274
|
+
f"HTTP {response.status_code}",
|
|
275
|
+
)
|
|
276
|
+
raise AssertionError("unreachable: the last attempt returns or raises") # pragma: no cover
|
|
277
|
+
|
|
278
|
+
def _wait(self, attempt: int, retry_after: float | None, why: str) -> None:
|
|
279
|
+
if retry_after is not None:
|
|
280
|
+
delay = min(retry_after, MAX_RETRY_AFTER_S)
|
|
281
|
+
else:
|
|
282
|
+
# Jittered so workers that failed together do not retry together.
|
|
283
|
+
delay = min(self._backoff_s * 2 ** (attempt - 1), MAX_BACKOFF_S) * random.uniform(
|
|
284
|
+
0.5, 1.0
|
|
285
|
+
)
|
|
286
|
+
log.warning(
|
|
287
|
+
"%s: %s on attempt %d/%d; retrying in %.2fs",
|
|
288
|
+
self._endpoint, why, attempt, self._max_attempts, delay,
|
|
289
|
+
) # fmt: skip
|
|
290
|
+
self._sleep(delay)
|
|
291
|
+
|
|
292
|
+
# ------------------------------------------------------------------ #
|
|
293
|
+
# Mapping
|
|
294
|
+
# ------------------------------------------------------------------ #
|
|
295
|
+
|
|
296
|
+
def _map(self, payload: dict[str, Any], session_id: str, latency_ms: float) -> ChatResponse:
|
|
297
|
+
m = self._mapping
|
|
298
|
+
|
|
299
|
+
text = _dig(payload, m["text"])
|
|
300
|
+
if text is _MISSING or not isinstance(text, str):
|
|
301
|
+
problem = "is missing" if text is _MISSING else f"is a {type(text).__name__}, not text"
|
|
302
|
+
raise AgentCallError(
|
|
303
|
+
f"response_mapping.text path {m['text']!r} {problem}; the payload's "
|
|
304
|
+
f"top-level keys are {sorted(payload)}"
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
route = _dig(payload, m["route"])
|
|
308
|
+
intent = _dig(payload, m["intent"])
|
|
309
|
+
session = _dig(payload, m["session"])
|
|
310
|
+
return ChatResponse(
|
|
311
|
+
reply=text,
|
|
312
|
+
conversation_id=session if isinstance(session, str) and session else session_id,
|
|
313
|
+
route=route if isinstance(route, str) and route else "unknown",
|
|
314
|
+
status=_status(_dig(payload, m["status"])),
|
|
315
|
+
intent=Intent(name=intent if isinstance(intent, str) and intent else "unknown"),
|
|
316
|
+
citations=[_citation(i, c) for i, c in enumerate(self._list(payload, "citations"))],
|
|
317
|
+
retrieved=[_chunk(i, c) for i, c in enumerate(self._list(payload, "retrieved"))],
|
|
318
|
+
tool_calls=[_tool(i, t) for i, t in enumerate(self._list(payload, "tools"))],
|
|
319
|
+
latency_ms=round(latency_ms, 3),
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
def _list(self, payload: dict[str, Any], key: str) -> list[Any]:
|
|
323
|
+
"""An optional list field: absent is empty, present-but-wrong is a contract break."""
|
|
324
|
+
value = _dig(payload, self._mapping[key])
|
|
325
|
+
if value is _MISSING or value is None:
|
|
326
|
+
return []
|
|
327
|
+
if not isinstance(value, list):
|
|
328
|
+
raise AgentCallError(
|
|
329
|
+
f"response_mapping.{key} path {self._mapping[key]!r} is a "
|
|
330
|
+
f"{type(value).__name__}, not a list"
|
|
331
|
+
)
|
|
332
|
+
return value
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
# --------------------------------------------------------------------------- #
|
|
336
|
+
# Request building
|
|
337
|
+
# --------------------------------------------------------------------------- #
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _validated_fields(
|
|
341
|
+
fields: Mapping[str, Mapping[str, Any]] | None,
|
|
342
|
+
principals: Mapping[str, Any],
|
|
343
|
+
template: Mapping[str, Any],
|
|
344
|
+
) -> dict[str, dict[str, Any]]:
|
|
345
|
+
"""Per-principal body fields, for agents that take identity in the payload.
|
|
346
|
+
|
|
347
|
+
Merged into the top level of the request body. When given at all, the labels
|
|
348
|
+
must be exactly those of `principal_headers`: a label with headers but no
|
|
349
|
+
fields would be sent with no identity in the body, and pass an isolation
|
|
350
|
+
case for the wrong reason. Use `{}` for a principal that needs none.
|
|
351
|
+
"""
|
|
352
|
+
if fields is None:
|
|
353
|
+
return {}
|
|
354
|
+
if set(fields) != set(principals):
|
|
355
|
+
raise ConfigError(
|
|
356
|
+
"agent.principal_fields and agent.principal_headers must declare the same "
|
|
357
|
+
f"principals; fields has {sorted(fields)}, headers has {sorted(principals)}"
|
|
358
|
+
)
|
|
359
|
+
out: dict[str, dict[str, Any]] = {}
|
|
360
|
+
for label, values in fields.items():
|
|
361
|
+
where = f"agent.principal_fields.{label}"
|
|
362
|
+
if not isinstance(values, Mapping):
|
|
363
|
+
raise ConfigError(f"{where} must be a mapping, got {type(values).__name__}")
|
|
364
|
+
for name, value in values.items():
|
|
365
|
+
# Same rule as headers: an unset ${VAR} expands to "", and the value
|
|
366
|
+
# is never echoed.
|
|
367
|
+
if not isinstance(name, str) or not name or value is None or value == "":
|
|
368
|
+
raise ConfigError(
|
|
369
|
+
f"{where}.{name} must be a non-empty value (is its environment variable set?)"
|
|
370
|
+
)
|
|
371
|
+
# Overwriting a template key could replace the message itself.
|
|
372
|
+
if name in template:
|
|
373
|
+
raise ConfigError(
|
|
374
|
+
f"{where}.{name} collides with a key of agent.request_template; "
|
|
375
|
+
"an identity field cannot overwrite the request"
|
|
376
|
+
)
|
|
377
|
+
out[label] = copy.deepcopy(dict(values))
|
|
378
|
+
return out
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _optional_mapping(agent: Mapping[str, Any], key: str) -> Mapping[str, Any] | None:
|
|
382
|
+
value = agent.get(key)
|
|
383
|
+
if value is not None and not isinstance(value, Mapping):
|
|
384
|
+
raise ConfigError(f"agent.{key} must be a mapping, got {value!r}")
|
|
385
|
+
return value
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _number(agent: Mapping[str, Any], key: str, default: float) -> float:
|
|
389
|
+
value = agent.get(key, default)
|
|
390
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
391
|
+
raise ConfigError(f"agent.{key} must be a number, got {value!r}")
|
|
392
|
+
return float(value)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
# --------------------------------------------------------------------------- #
|
|
396
|
+
# Response reading
|
|
397
|
+
# --------------------------------------------------------------------------- #
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _json_object(response: httpx.Response) -> dict[str, Any]:
|
|
401
|
+
try:
|
|
402
|
+
payload = response.json()
|
|
403
|
+
except ValueError as exc:
|
|
404
|
+
raise AgentCallError(
|
|
405
|
+
f"response was not JSON (HTTP {response.status_code}, "
|
|
406
|
+
f"content-type {response.headers.get('content-type', 'none')!r}): {_snippet(response)}"
|
|
407
|
+
) from exc
|
|
408
|
+
if not isinstance(payload, dict):
|
|
409
|
+
raise AgentCallError(
|
|
410
|
+
f"response JSON must be an object, got {type(payload).__name__}"
|
|
411
|
+
)
|
|
412
|
+
return payload
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _retry_after(value: str | None) -> float | None:
|
|
416
|
+
"""Seconds from a `Retry-After` header (a number, or an HTTP date), else None."""
|
|
417
|
+
if not value:
|
|
418
|
+
return None
|
|
419
|
+
try:
|
|
420
|
+
return max(0.0, float(value))
|
|
421
|
+
except ValueError:
|
|
422
|
+
pass
|
|
423
|
+
try:
|
|
424
|
+
when = email.utils.parsedate_to_datetime(value)
|
|
425
|
+
except (TypeError, ValueError):
|
|
426
|
+
return None
|
|
427
|
+
if when.tzinfo is None:
|
|
428
|
+
when = when.replace(tzinfo=UTC)
|
|
429
|
+
return max(0.0, (when - datetime.now(UTC)).total_seconds())
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _snippet(response: httpx.Response) -> str:
|
|
433
|
+
"""A short, single-line look at a body, for an error message."""
|
|
434
|
+
text = " ".join(response.text.split())
|
|
435
|
+
return text[:200] + ("..." if len(text) > 200 else "") or "(empty body)"
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
__all__ = [
|
|
439
|
+
"DEFAULT_REQUEST_TEMPLATE",
|
|
440
|
+
"DEFAULT_RESPONSE_MAPPING",
|
|
441
|
+
"RETRYABLE_STATUSES",
|
|
442
|
+
"HTTPAdapter",
|
|
443
|
+
]
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""The in-process adapter: the simulator, reached through the adapter seam.
|
|
2
|
+
|
|
3
|
+
This wraps an engine living in this process - the simulator's
|
|
4
|
+
`ConversationEngine` - so the harness can drive it the same way it will drive a
|
|
5
|
+
deployed agent. The adapter knows the engine only as a shape (`InProcessEngine`)
|
|
6
|
+
and imports nothing from the simulator: whoever has one passes it in. Nothing
|
|
7
|
+
here is transport:
|
|
8
|
+
there is no client to open and no payload to map, because the engine already
|
|
9
|
+
speaks `ChatResponse`. What the adapter does own is identity - turning the
|
|
10
|
+
principal label a suite names into the user and tenant the engine scopes
|
|
11
|
+
retrieval by.
|
|
12
|
+
|
|
13
|
+
It stands in for the real system until the HTTP adapter exists. Numbers produced
|
|
14
|
+
through it are results about the simulator, and `describe()` says so.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from collections.abc import Mapping
|
|
20
|
+
from typing import Protocol
|
|
21
|
+
|
|
22
|
+
from ..core.contracts import ChatRequest, ChatResponse
|
|
23
|
+
from ..core.errors import ConfigError
|
|
24
|
+
from .base import AdapterLifecycle
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class InProcessEngine(Protocol):
|
|
28
|
+
"""What this adapter needs of an engine. The simulator's fits; so can a fake."""
|
|
29
|
+
|
|
30
|
+
def chat(self, request: ChatRequest) -> ChatResponse:
|
|
31
|
+
"""Answer one turn."""
|
|
32
|
+
...
|
|
33
|
+
|
|
34
|
+
def provenance(self) -> dict[str, str]:
|
|
35
|
+
"""Which engine this is, for the run report. Strings, stable keys."""
|
|
36
|
+
...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
#: principal label -> (user_id, tenant_id). "default" is the Protocol's own
|
|
40
|
+
#: default for `send`, so it has to resolve; it is the same identity suites have
|
|
41
|
+
#: always run as.
|
|
42
|
+
DEFAULT_PRINCIPALS: Mapping[str, tuple[str, str]] = {
|
|
43
|
+
"default": ("test_user", "default"),
|
|
44
|
+
"test_user": ("test_user", "default"),
|
|
45
|
+
"outsider": ("outsider", "default"),
|
|
46
|
+
"acme_user": ("acme_user", "acme"),
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class InProcessAdapter(AdapterLifecycle):
|
|
51
|
+
"""`AgentAdapter` over a `ConversationEngine` living in this process."""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
engine: InProcessEngine,
|
|
56
|
+
principals: Mapping[str, tuple[str, str]] | None = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
super().__init__()
|
|
59
|
+
self._engine = engine
|
|
60
|
+
self._principals = _validated(
|
|
61
|
+
DEFAULT_PRINCIPALS if principals is None else principals
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def send(
|
|
65
|
+
self, message: str, *, session_id: str, principal: str = "default"
|
|
66
|
+
) -> ChatResponse:
|
|
67
|
+
"""Send one turn to the engine and return its response untouched.
|
|
68
|
+
|
|
69
|
+
No lock: the runner's thread pool already calls `engine.chat` from
|
|
70
|
+
several workers at once, so the adapter shares nothing that was not
|
|
71
|
+
shared before it existed.
|
|
72
|
+
"""
|
|
73
|
+
self._require_open("send")
|
|
74
|
+
user_id, tenant_id = self._resolve(principal)
|
|
75
|
+
return self._engine.chat(
|
|
76
|
+
ChatRequest(
|
|
77
|
+
message=message,
|
|
78
|
+
conversation_id=session_id,
|
|
79
|
+
user_id=user_id,
|
|
80
|
+
tenant_id=tenant_id,
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def describe(self) -> dict[str, str]:
|
|
85
|
+
"""Provenance for the run report.
|
|
86
|
+
|
|
87
|
+
The engine states its own keys - the ones the runner has always
|
|
88
|
+
recorded, so a baseline taken before the adapter existed still lines up.
|
|
89
|
+
`adapter` is what stops a simulator baseline being read as comparable to
|
|
90
|
+
a run against a deployed agent, so the engine cannot overwrite it.
|
|
91
|
+
"""
|
|
92
|
+
self._require_open("describe")
|
|
93
|
+
info = {"adapter": "in_process", **self._engine.provenance()}
|
|
94
|
+
info["adapter"] = "in_process"
|
|
95
|
+
return info
|
|
96
|
+
|
|
97
|
+
def _resolve(self, principal: str) -> tuple[str, str]:
|
|
98
|
+
"""Map a principal label to (user_id, tenant_id), or refuse.
|
|
99
|
+
|
|
100
|
+
An unknown label never falls back to a default identity: an isolation
|
|
101
|
+
case that quietly ran as the wrong user would pass for the wrong reason.
|
|
102
|
+
"""
|
|
103
|
+
try:
|
|
104
|
+
return self._principals[principal]
|
|
105
|
+
except KeyError:
|
|
106
|
+
raise ConfigError(
|
|
107
|
+
f"unknown principal {principal!r}; this adapter knows: "
|
|
108
|
+
f"{sorted(self._principals)}"
|
|
109
|
+
) from None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _validated(principals: Mapping[str, tuple[str, str]]) -> dict[str, tuple[str, str]]:
|
|
113
|
+
"""Copy the mapping, refusing any value that is not a (user_id, tenant_id) pair.
|
|
114
|
+
|
|
115
|
+
Checked when the adapter is built so a malformed entry stops the run before
|
|
116
|
+
it starts, rather than failing the first case sent as that principal. The
|
|
117
|
+
copy means a caller mutating its mapping mid-run cannot change who a later
|
|
118
|
+
turn is sent as.
|
|
119
|
+
"""
|
|
120
|
+
for label, identity in principals.items():
|
|
121
|
+
# `tuple` is checked by type: a two-character string would otherwise
|
|
122
|
+
# unpack into a user and a tenant without complaint.
|
|
123
|
+
if not (
|
|
124
|
+
isinstance(identity, tuple)
|
|
125
|
+
and len(identity) == 2
|
|
126
|
+
and all(isinstance(part, str) for part in identity)
|
|
127
|
+
):
|
|
128
|
+
raise ConfigError(
|
|
129
|
+
f"principal {label!r} must map to a (user_id, tenant_id) tuple of "
|
|
130
|
+
f"two strings, got {identity!r}"
|
|
131
|
+
)
|
|
132
|
+
return dict(principals)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
__all__ = ["DEFAULT_PRINCIPALS", "InProcessAdapter", "InProcessEngine"]
|