askback 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. askback/__init__.py +29 -0
  2. askback/__main__.py +8 -0
  3. askback/adapters/__init__.py +11 -0
  4. askback/adapters/base.py +189 -0
  5. askback/adapters/factory.py +176 -0
  6. askback/adapters/http.py +443 -0
  7. askback/adapters/in_process.py +135 -0
  8. askback/adapters/payload.py +207 -0
  9. askback/adapters/scripted.py +117 -0
  10. askback/adapters/socketio_adapter.py +885 -0
  11. askback/api/__init__.py +18 -0
  12. askback/api/corpus_store.py +85 -0
  13. askback/api/dashboard.py +444 -0
  14. askback/api/server.py +200 -0
  15. askback/api/state.py +82 -0
  16. askback/api/static/app.css +423 -0
  17. askback/api/static/app.js +98 -0
  18. askback/api/static/core.js +201 -0
  19. askback/api/static/index.html +175 -0
  20. askback/api/static/panels.js +651 -0
  21. askback/cli.py +618 -0
  22. askback/core/__init__.py +58 -0
  23. askback/core/config.py +224 -0
  24. askback/core/contracts.py +201 -0
  25. askback/core/errors.py +35 -0
  26. askback/core/logging.py +57 -0
  27. askback/core/registry.py +73 -0
  28. askback/core/text.py +151 -0
  29. askback/datasets/__init__.py +14 -0
  30. askback/datasets/loader.py +268 -0
  31. askback/datasets/schema.py +525 -0
  32. askback/demo/__init__.py +42 -0
  33. askback/demo/agent.py +195 -0
  34. askback/demo/suites/demo.yaml +100 -0
  35. askback/evaluation/__init__.py +70 -0
  36. askback/evaluation/base.py +214 -0
  37. askback/evaluation/calibration.py +405 -0
  38. askback/evaluation/citation_eval.py +580 -0
  39. askback/evaluation/clarification.py +477 -0
  40. askback/evaluation/engine.py +193 -0
  41. askback/evaluation/escalation_eval.py +109 -0
  42. askback/evaluation/gates.py +351 -0
  43. askback/evaluation/judge_cache.py +94 -0
  44. askback/evaluation/llm_judge.py +463 -0
  45. askback/evaluation/patterns.py +48 -0
  46. askback/evaluation/pii_eval.py +252 -0
  47. askback/evaluation/rag_eval.py +168 -0
  48. askback/evaluation/regression.py +438 -0
  49. askback/evaluation/results.py +588 -0
  50. askback/evaluation/rule_based.py +198 -0
  51. askback/evaluation/safety_eval.py +170 -0
  52. askback/providers/__init__.py +10 -0
  53. askback/providers/base.py +157 -0
  54. askback/providers/openai_compat.py +340 -0
  55. askback/py.typed +0 -0
  56. askback/reporting/__init__.py +77 -0
  57. askback/reporting/excel_report.py +117 -0
  58. askback/reporting/html_report.py +283 -0
  59. askback/reporting/json_report.py +27 -0
  60. askback/reporting/markdown_report.py +95 -0
  61. askback/reporting/redaction.py +185 -0
  62. askback/runner/__init__.py +5 -0
  63. askback/runner/responders.py +279 -0
  64. askback/runner/test_runner.py +424 -0
  65. askback-0.1.0.dist-info/METADATA +504 -0
  66. askback-0.1.0.dist-info/RECORD +71 -0
  67. askback-0.1.0.dist-info/WHEEL +5 -0
  68. askback-0.1.0.dist-info/entry_points.txt +2 -0
  69. askback-0.1.0.dist-info/licenses/LICENSE +176 -0
  70. askback-0.1.0.dist-info/licenses/NOTICE +15 -0
  71. askback-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,885 @@
1
+ """The Socket.IO adapter: an agent that answers over a long-lived event connection.
2
+
3
+ Socket.IO is not raw WebSocket. It has its own HTTP handshake and upgrade, its own
4
+ packet framing, namespaces and heartbeats, so a plain websocket client fails before
5
+ any application frame exists. This module uses `python-socketio`, which speaks the
6
+ protocol, and adds only what a test harness needs on top of it: which events are
7
+ which, when a reply is complete, who we are, and how fast we may go.
8
+
9
+ Needs `python-socketio[client]`, from the `socketio` extra. Nothing imports this
10
+ module until `agent.adapter: socketio` asks for it, and a missing library is a
11
+ `ConfigError` with the install command, not an `ImportError`.
12
+
13
+ Events. An agent's event names are its own, so none is hardcoded: config says which
14
+ are content (carry reply text), metadata (carry routing, tools, citations, and
15
+ arrive on their own events, usually before the text), the terminator (the reply is
16
+ complete), and errors. Every other event is ignorable and is logged once by name.
17
+ Every classified event's payload is merged and read with the same `response_mapping`
18
+ the HTTP adapter uses, so metadata is harvested from all of them, not just the one
19
+ that carried the text.
20
+
21
+ One connection per session. A session's connection is opened by its first turn and
22
+ reused for the rest, and it is never reopened. A reconnect starts a new server-side
23
+ conversation without the history, which shows up in a report as the agent forgetting
24
+ - a memory bug that is really ours. So a connection that is lost takes its session
25
+ with it: every later turn of that session fails and says why, and new sessions are
26
+ unaffected.
27
+
28
+ Retries, and what they cost. Only opening a connection is retried: nothing has been
29
+ sent, so trying again cannot deliver anything twice. Once a turn has been emitted it
30
+ is NEVER sent again, whatever happens next (timeout, disconnect, error event): the
31
+ agent may already have acted on it, and a replay would be answered by an agent that
32
+ now holds the turn twice. This is the opposite of what `HTTPAdapter` does for a
33
+ retried POST, deliberately.
34
+
35
+ Completion. A reply ends at the terminator event. A server with none needs an
36
+ explicit `idle_timeout_s`: the reply is complete once that long passes after its
37
+ last event. There is no default for it, because a short guess would cut a reply that
38
+ pauses mid-stream and grade the fragment. `timeout_s` (120 by default; replies on
39
+ this class of system run past 45s) bounds the whole turn, and is where a hung agent
40
+ ends up, as an error and not as a score.
41
+
42
+ Rate limits. `rate_limit` is a list of sliding windows, and `send` waits for room in
43
+ all of them before emitting. The limiter is shared by every worker of the adapter.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import base64
49
+ import binascii
50
+ import importlib
51
+ import json
52
+ import queue
53
+ import threading
54
+ import time
55
+ from collections import deque
56
+ from collections.abc import Callable, Mapping, Sequence
57
+ from concurrent.futures import ThreadPoolExecutor
58
+ from dataclasses import dataclass, field
59
+ from types import ModuleType
60
+ from typing import Any, Protocol
61
+ from urllib.parse import urlsplit
62
+
63
+ from ..core.contracts import ChatResponse, Intent
64
+ from ..core.errors import AgentCallError, ConfigError
65
+ from ..core.logging import get_logger
66
+ from .base import AdapterLifecycle
67
+ from .payload import (
68
+ MISSING,
69
+ chunk,
70
+ citation,
71
+ dig,
72
+ fill,
73
+ status_from,
74
+ tool,
75
+ validated_headers,
76
+ validated_mapping,
77
+ validated_template,
78
+ )
79
+
80
+ log = get_logger(__name__)
81
+
82
+ INSTALL_HINT = "agent.adapter 'socketio' needs python-socketio. Run: pip install -e .[socketio]"
83
+
84
+ DEFAULT_REQUEST_TEMPLATE: Mapping[str, Any] = {"message": "{message}"}
85
+
86
+ #: Longest the computed backoff between connection attempts grows to.
87
+ MAX_BACKOFF_S = 30.0
88
+
89
+ _DISCONNECT = "\0disconnect"
90
+ _CONTENT_MODES = ("append", "replace")
91
+
92
+
93
+ # --------------------------------------------------------------------------- #
94
+ # Authentication
95
+ # --------------------------------------------------------------------------- #
96
+
97
+
98
+ @dataclass(frozen=True, slots=True)
99
+ class Credentials:
100
+ """What a connection presents when it opens: HTTP headers and/or a connect payload."""
101
+
102
+ headers: Mapping[str, str] = field(default_factory=dict)
103
+ #: The `auth` object of the Socket.IO CONNECT packet, for servers that read it there.
104
+ auth: Mapping[str, Any] | None = None
105
+
106
+
107
+ class ConnectionAuth(Protocol):
108
+ """Supplies credentials for a principal, and can be told they were refused.
109
+
110
+ Called every time a connection opens, so a token that expired mid-run is
111
+ replaced for the next session without the runner knowing. `refresh=True` means
112
+ the credentials last returned were refused: do not hand out the same ones.
113
+ Implementations must be thread-safe; workers open connections concurrently.
114
+ """
115
+
116
+ def credentials(self, principal: str, *, refresh: bool = False) -> Credentials: ...
117
+
118
+
119
+ class StaticAuth:
120
+ """Fixed credentials per principal label, from config. Never falls back to anonymous."""
121
+
122
+ def __init__(
123
+ self,
124
+ principal_headers: Mapping[str, Mapping[str, str]],
125
+ principal_auth: Mapping[str, Mapping[str, Any]] | None = None,
126
+ ) -> None:
127
+ self._headers = {
128
+ label: validated_headers(value, f"agent.principal_headers.{label}")
129
+ for label, value in principal_headers.items()
130
+ }
131
+ self._auth = {label: dict(value) for label, value in (principal_auth or {}).items()}
132
+ for label in self._auth:
133
+ if label not in self._headers:
134
+ raise ConfigError(
135
+ f"agent.principal_auth.{label} has no matching agent.principal_headers entry; "
136
+ "use `{}` there for a principal that needs no header"
137
+ )
138
+
139
+ def credentials(self, principal: str, *, refresh: bool = False) -> Credentials:
140
+ try:
141
+ headers = self._headers[principal]
142
+ except KeyError:
143
+ raise ConfigError(
144
+ f"unknown principal {principal!r}; agent.principal_headers knows: "
145
+ f"{sorted(self._headers)}"
146
+ ) from None
147
+ return Credentials(headers=headers, auth=self._auth.get(principal))
148
+
149
+
150
+ @dataclass(frozen=True, slots=True)
151
+ class Token:
152
+ value: str
153
+ #: Epoch seconds, or None when the token says nothing about its own expiry.
154
+ expires_at: float | None = None
155
+
156
+
157
+ def jwt_expiry(token: str) -> float | None:
158
+ """The `exp` claim of a JWT, read WITHOUT verifying it. None if there is none.
159
+
160
+ Only used to decide when to ask for a new token, never to trust one.
161
+ """
162
+ parts = token.split(".")
163
+ if len(parts) != 3:
164
+ return None
165
+ try:
166
+ padded = parts[1] + "=" * (-len(parts[1]) % 4)
167
+ claims = json.loads(base64.urlsafe_b64decode(padded))
168
+ except (ValueError, binascii.Error):
169
+ return None
170
+ exp = claims.get("exp") if isinstance(claims, dict) else None
171
+ return float(exp) if isinstance(exp, (int, float)) and not isinstance(exp, bool) else None
172
+
173
+
174
+ class CachedTokenAuth:
175
+ """Base for an auth hook that logs in (signin, OTP, ...) and keeps the token fresh.
176
+
177
+ Subclass it and write `login`; that is where the signin/OTP exchange lives.
178
+ Override `refresh` to use a refresh token instead of logging in again. The token
179
+ is cached per principal and replaced shortly before it expires, or at once when
180
+ the adapter says it was refused (`refresh=True`). One lock serialises logins:
181
+ they are rare, and two workers must not run an OTP flow for one principal twice.
182
+ A failure raises `AgentCallError`.
183
+ """
184
+
185
+ header = "Authorization"
186
+ scheme = "Bearer"
187
+ #: Renew this long before the token's own expiry, so it is never presented dead.
188
+ skew_s = 30.0
189
+
190
+ def __init__(self, *, clock: Callable[[], float] = time.time) -> None:
191
+ self._clock = clock
192
+ self._tokens: dict[str, Token] = {}
193
+ self._lock = threading.Lock()
194
+
195
+ def login(self, principal: str) -> Token:
196
+ raise NotImplementedError("a CachedTokenAuth subclass implements login()")
197
+
198
+ def refresh(self, principal: str, stale: Token) -> Token:
199
+ return self.login(principal)
200
+
201
+ def credentials(self, principal: str, *, refresh: bool = False) -> Credentials:
202
+ with self._lock:
203
+ token = self._tokens.get(principal)
204
+ if token is None:
205
+ token = self.login(principal)
206
+ elif refresh or self._expired(token):
207
+ try:
208
+ token = self.refresh(principal, token)
209
+ except AgentCallError as exc:
210
+ log.warning("Token refresh for %r failed (%s); logging in again.", principal, exc)
211
+ token = self.login(principal)
212
+ self._tokens[principal] = token
213
+ value = f"{self.scheme} {token.value}" if self.scheme else token.value
214
+ return Credentials(headers={self.header: value})
215
+
216
+ def _expired(self, token: Token) -> bool:
217
+ expires = token.expires_at if token.expires_at is not None else jwt_expiry(token.value)
218
+ return expires is not None and self._clock() >= expires - self.skew_s
219
+
220
+
221
+ def load_auth(spec: Mapping[str, Any]) -> ConnectionAuth:
222
+ """Build the hook named by `agent.auth`: `{hook: "package.module:factory", options: {...}}`.
223
+
224
+ The factory is called with `options` as keyword arguments. It is code named by
225
+ your own config file, run with your own privileges, exactly like a plugin.
226
+ """
227
+ stray = sorted(set(spec) - {"hook", "options"})
228
+ if stray:
229
+ raise ConfigError(f"unknown agent.auth key(s): {stray}; known: hook, options")
230
+ hook = spec.get("hook")
231
+ if not isinstance(hook, str) or hook.count(":") != 1 or not all(hook.split(":")):
232
+ raise ConfigError(f"agent.auth.hook must be 'package.module:factory', got {hook!r}")
233
+ options = spec.get("options") or {}
234
+ if not isinstance(options, Mapping):
235
+ raise ConfigError(f"agent.auth.options must be a mapping, got {options!r}")
236
+ module_name, _, factory_name = hook.partition(":")
237
+ try:
238
+ factory = getattr(importlib.import_module(module_name), factory_name)
239
+ except (ImportError, AttributeError) as exc:
240
+ raise ConfigError(f"agent.auth.hook {hook!r} could not be loaded: {exc}") from exc
241
+ try:
242
+ auth: ConnectionAuth = factory(**options)
243
+ except TypeError as exc:
244
+ raise ConfigError(f"agent.auth.hook {hook!r} does not fit its options: {exc}") from exc
245
+ if not callable(getattr(auth, "credentials", None)):
246
+ raise ConfigError(f"agent.auth.hook {hook!r} returned something with no `credentials()`")
247
+ return auth
248
+
249
+
250
+ # --------------------------------------------------------------------------- #
251
+ # Rate limiting
252
+ # --------------------------------------------------------------------------- #
253
+
254
+
255
+ class RateLimiter:
256
+ """Sliding windows of (max requests, per seconds). `acquire` waits until all have room.
257
+
258
+ Thread-safe. A request is counted when it is let through, so concurrent
259
+ workers share one budget and the agent never sees more than the windows allow.
260
+ """
261
+
262
+ def __init__(
263
+ self,
264
+ windows: Sequence[tuple[int, float]],
265
+ *,
266
+ clock: Callable[[], float] = time.monotonic,
267
+ sleep: Callable[[float], None] = time.sleep,
268
+ ) -> None:
269
+ self._windows = [(count, per_s, deque[float]()) for count, per_s in windows]
270
+ self._clock = clock
271
+ self._sleep = sleep
272
+ self._lock = threading.Lock()
273
+
274
+ def acquire(self) -> float:
275
+ """Block until a request may go, and return how long that took, in seconds."""
276
+ waited = 0.0
277
+ while True:
278
+ with self._lock:
279
+ now = self._clock()
280
+ wait = 0.0
281
+ for count, per_s, stamps in self._windows:
282
+ while stamps and now - stamps[0] >= per_s:
283
+ stamps.popleft()
284
+ if len(stamps) >= count:
285
+ wait = max(wait, per_s - (now - stamps[0]))
286
+ if wait <= 0:
287
+ for _, _, stamps in self._windows:
288
+ stamps.append(now)
289
+ return waited
290
+ self._sleep(wait)
291
+ waited += wait
292
+
293
+
294
+ # --------------------------------------------------------------------------- #
295
+ # The adapter
296
+ # --------------------------------------------------------------------------- #
297
+
298
+
299
+ @dataclass(frozen=True, slots=True)
300
+ class Events:
301
+ """Which event names mean what. Disjoint by construction."""
302
+
303
+ send: str
304
+ content: frozenset[str]
305
+ metadata: frozenset[str] = frozenset()
306
+ terminator: str | None = None
307
+ errors: frozenset[str] = frozenset()
308
+
309
+ def __post_init__(self) -> None:
310
+ named = [*self.content, *self.metadata, *self.errors]
311
+ if self.terminator is not None:
312
+ named.append(self.terminator)
313
+ clash = sorted({n for n in named if named.count(n) > 1})
314
+ if clash:
315
+ raise ConfigError(f"agent.events: {clash} are given more than one role")
316
+
317
+ def classify(self, event: str) -> str:
318
+ if event in self.errors:
319
+ return "error"
320
+ if event == self.terminator:
321
+ return "terminator"
322
+ if event in self.content:
323
+ return "content"
324
+ if event in self.metadata:
325
+ return "metadata"
326
+ return "ignorable"
327
+
328
+
329
+ class _Conn:
330
+ """One session's connection, and the events it has received and not yet used."""
331
+
332
+ def __init__(self, principal: str) -> None:
333
+ self.principal = principal
334
+ self.client: Any = None
335
+ self.inbox: queue.Queue[tuple[str, Any]] = queue.Queue()
336
+ self.turn_lock = threading.Lock()
337
+ #: Why the connection is gone, once it is. Set from the client's own thread.
338
+ self.lost: str | None = None
339
+
340
+
341
+ class SocketIOAdapter(AdapterLifecycle):
342
+ """`AgentAdapter` over a Socket.IO namespace: one connection per session, never replayed."""
343
+
344
+ def __init__(
345
+ self,
346
+ url: str,
347
+ *,
348
+ events: Events,
349
+ auth: ConnectionAuth,
350
+ namespace: str = "/",
351
+ socketio_path: str = "socket.io",
352
+ transports: Sequence[str] | None = None,
353
+ request_template: Mapping[str, Any] | None = None,
354
+ response_mapping: Mapping[str, str] | None = None,
355
+ content_mode: str = "append",
356
+ timeout_s: float = 120.0,
357
+ idle_timeout_s: float | None = None,
358
+ connect_timeout_s: float = 10.0,
359
+ max_attempts: int = 3,
360
+ backoff_s: float = 0.5,
361
+ rate_limit: Sequence[tuple[int, float]] = (),
362
+ sleep: Callable[[float], None] = time.sleep,
363
+ limiter: RateLimiter | None = None,
364
+ ) -> None:
365
+ super().__init__()
366
+ try:
367
+ self._sio: ModuleType = importlib.import_module("socketio")
368
+ except ImportError as exc:
369
+ raise ConfigError(INSTALL_HINT) from exc
370
+ parts = urlsplit(url)
371
+ if parts.scheme not in ("http", "https", "ws", "wss") or not parts.netloc:
372
+ raise ConfigError(f"agent.url must be an http(s) or ws(s) URL, got {url!r}")
373
+ if not namespace.startswith("/"):
374
+ raise ConfigError(f"agent.namespace must start with '/', got {namespace!r}")
375
+ if content_mode not in _CONTENT_MODES:
376
+ raise ConfigError(f"agent.content_mode must be one of {_CONTENT_MODES}, got {content_mode!r}")
377
+ for name, value in (("timeout_s", timeout_s), ("connect_timeout_s", connect_timeout_s)):
378
+ if value <= 0:
379
+ raise ConfigError(f"agent.{name} must be positive, got {value!r}")
380
+ if idle_timeout_s is not None and idle_timeout_s <= 0:
381
+ raise ConfigError(f"agent.idle_timeout_s must be positive, got {idle_timeout_s!r}")
382
+ if events.terminator is None and idle_timeout_s is None:
383
+ raise ConfigError(
384
+ "agent has no way to tell a reply is complete: set agent.events.terminator, "
385
+ "or agent.idle_timeout_s (seconds of quiet that end a reply). There is no "
386
+ "default: a short guess would cut a reply that pauses and grade the fragment."
387
+ )
388
+ if max_attempts < 1 or backoff_s < 0:
389
+ raise ConfigError("agent.max_attempts must be >= 1 and agent.backoff_s >= 0")
390
+
391
+ self._url = url
392
+ self._endpoint = f"{parts.scheme}://{parts.hostname}{f':{parts.port}' if parts.port else ''}{namespace}"
393
+ self._namespace = namespace
394
+ self._socketio_path = socketio_path
395
+ self._transports = list(transports) if transports else None
396
+ self._events = events
397
+ self._auth = auth
398
+ self._template = validated_template(request_template or DEFAULT_REQUEST_TEMPLATE)
399
+ self._mapping = validated_mapping(response_mapping)
400
+ self._content_mode = content_mode
401
+ self._timeout_s = timeout_s
402
+ self._idle_timeout_s = idle_timeout_s
403
+ self._connect_timeout_s = connect_timeout_s
404
+ self._max_attempts = max_attempts
405
+ self._backoff_s = backoff_s
406
+ self._sleep = sleep
407
+ self._limiter = limiter or RateLimiter(list(rate_limit))
408
+ self._conns: dict[str, _Conn] = {}
409
+ self._conns_lock = threading.Lock()
410
+ self._unclassified: set[str] = set()
411
+
412
+ @classmethod
413
+ def from_config(cls, agent: Mapping[str, Any]) -> SocketIOAdapter:
414
+ """Build from the `agent:` section of your config file."""
415
+ known = {
416
+ "adapter", "url", "namespace", "socketio_path", "transports", "events",
417
+ "request_template", "response_mapping", "content_mode", "timeout_s",
418
+ "idle_timeout_s", "connect_timeout_s", "max_attempts", "backoff_s", "rate_limit",
419
+ "principal_headers", "principal_auth", "auth",
420
+ # Read by the factory; named here so the stray-key check lets it through.
421
+ "side_effecting_tools",
422
+ } # fmt: skip
423
+ stray = sorted(set(agent) - known)
424
+ if stray:
425
+ raise ConfigError(
426
+ f"unknown agent key(s) for the socketio adapter: {stray}; known: {sorted(known)}"
427
+ )
428
+ url = agent.get("url")
429
+ if not isinstance(url, str) or not url:
430
+ raise ConfigError("agent.url is required for the socketio adapter")
431
+ return cls(
432
+ url,
433
+ events=_events_from(agent.get("events")),
434
+ auth=_auth_from(agent),
435
+ namespace=_text(agent, "namespace", "/"),
436
+ socketio_path=_text(agent, "socketio_path", "socket.io"),
437
+ transports=_str_list(agent, "transports") or None,
438
+ request_template=_optional_mapping(agent, "request_template"),
439
+ response_mapping=_optional_mapping(agent, "response_mapping"),
440
+ content_mode=_text(agent, "content_mode", "append"),
441
+ timeout_s=_float(agent, "timeout_s", 120.0),
442
+ idle_timeout_s=_number(agent, "idle_timeout_s", None),
443
+ connect_timeout_s=_float(agent, "connect_timeout_s", 10.0),
444
+ max_attempts=int(_float(agent, "max_attempts", 3.0)),
445
+ backoff_s=_float(agent, "backoff_s", 0.5),
446
+ rate_limit=_rate_limit_from(agent.get("rate_limit")),
447
+ )
448
+
449
+ # ------------------------------------------------------------------ #
450
+ # Lifecycle: connections live for the run, one per session.
451
+ # ------------------------------------------------------------------ #
452
+
453
+ def _close(self) -> None:
454
+ with self._conns_lock:
455
+ conns, self._conns = list(self._conns.values()), {}
456
+ clients = []
457
+ for conn in conns:
458
+ client, conn.client = conn.client, None
459
+ if client is not None:
460
+ clients.append(client)
461
+ if not clients:
462
+ return
463
+ # In parallel: a close can wait on the server (seconds, on some), and a run
464
+ # holds one connection per session.
465
+ with ThreadPoolExecutor(max_workers=min(16, len(clients))) as pool:
466
+ list(pool.map(self._disconnect, clients))
467
+
468
+ @staticmethod
469
+ def _disconnect(client: Any) -> None:
470
+ try:
471
+ client.disconnect()
472
+ except Exception: # noqa: BLE001 - documented boundary: closing must not mask the run's own error
473
+ log.exception("Socket.IO disconnect failed while closing the adapter")
474
+
475
+ def describe(self) -> dict[str, str]:
476
+ """The endpoint and namespace, without credentials or a query string."""
477
+ self._require_open("describe")
478
+ return {"adapter": "socketio", "endpoint": self._endpoint}
479
+
480
+ # ------------------------------------------------------------------ #
481
+ # AgentAdapter
482
+ # ------------------------------------------------------------------ #
483
+
484
+ def send(
485
+ self, message: str, *, session_id: str, principal: str = "default"
486
+ ) -> ChatResponse:
487
+ """Emit one turn on the session's connection and collect the reply it provokes.
488
+
489
+ Opens the session's connection on its first turn. Never re-sends a turn:
490
+ any failure after the emit is raised, not retried.
491
+ """
492
+ self._require_open("send")
493
+ conn = self._conn_for(session_id, principal)
494
+ with conn.turn_lock:
495
+ if conn.client is None and conn.lost is None:
496
+ self._connect(conn, session_id)
497
+ self._require_alive(conn, session_id)
498
+
499
+ body = fill(self._template, {"message": message, "session_id": session_id})
500
+ self._limiter.acquire()
501
+ stale = self._drain(conn)
502
+ if stale:
503
+ log.debug("Session %s: discarded %d late event(s) from the last turn", session_id, stale)
504
+
505
+ started = time.perf_counter()
506
+ try:
507
+ conn.client.emit(self._events.send, body, namespace=self._namespace)
508
+ except Exception as exc: # noqa: BLE001 - the library raises several types; delivery is unknown either way
509
+ conn.lost = f"emit failed: {type(exc).__name__}: {exc}"
510
+ raise AgentCallError(
511
+ f"{self._endpoint} could not take the turn for session {session_id!r} "
512
+ f"({type(exc).__name__}); not retried, the session is lost"
513
+ ) from exc
514
+
515
+ merged, texts = self._await_reply(conn, session_id)
516
+ latency_ms = (time.perf_counter() - started) * 1000
517
+ return self._map(merged, texts, session_id, latency_ms)
518
+
519
+ # ------------------------------------------------------------------ #
520
+ # Connections
521
+ # ------------------------------------------------------------------ #
522
+
523
+ def _conn_for(self, session_id: str, principal: str) -> _Conn:
524
+ with self._conns_lock:
525
+ conn = self._conns.get(session_id)
526
+ if conn is None:
527
+ conn = self._conns[session_id] = _Conn(principal)
528
+ if conn.principal != principal:
529
+ raise ConfigError(
530
+ f"session {session_id!r} was opened as principal {conn.principal!r} and cannot "
531
+ f"continue as {principal!r}: one connection is one identity"
532
+ )
533
+ return conn
534
+
535
+ def _require_alive(self, conn: _Conn, session_id: str) -> None:
536
+ if conn.lost is None and conn.client is not None and not conn.client.connected:
537
+ conn.lost = "the connection dropped between turns"
538
+ if conn.lost is not None:
539
+ raise AgentCallError(
540
+ f"{self._endpoint} session {session_id!r} lost its connection ({conn.lost}). "
541
+ "No reconnect was made: a new connection is a new conversation without this "
542
+ "one's history, and would read as the agent forgetting."
543
+ )
544
+
545
+ def _connect(self, conn: _Conn, session_id: str) -> None:
546
+ """Open the session's connection. Safe to retry: nothing has been sent on it."""
547
+ sio = self._sio
548
+ refresh = False
549
+ for attempt in range(1, self._max_attempts + 1):
550
+ credentials = self._auth.credentials(conn.principal, refresh=refresh)
551
+ # reconnection=False: the library would otherwise reconnect on its own,
552
+ # silently starting the conversation over.
553
+ client = sio.Client(reconnection=False, logger=False, engineio_logger=False)
554
+ client.on("*", self._catch_all(conn), namespace=self._namespace)
555
+ client.on("disconnect", self._on_disconnect(conn), namespace=self._namespace)
556
+ try:
557
+ client.connect(
558
+ self._url,
559
+ headers=dict(credentials.headers),
560
+ auth=dict(credentials.auth) if credentials.auth is not None else None,
561
+ namespaces=[self._namespace],
562
+ transports=self._transports,
563
+ socketio_path=self._socketio_path,
564
+ wait_timeout=self._connect_timeout_s,
565
+ )
566
+ except sio.exceptions.SocketIOError as exc:
567
+ self._discard(client)
568
+ if attempt == self._max_attempts:
569
+ raise AgentCallError(
570
+ f"{self._endpoint} refused or dropped the connection for session "
571
+ f"{session_id!r} after {attempt} attempt(s): {type(exc).__name__}: {exc}"
572
+ ) from exc
573
+ # A refusal may be an expired token: ask for fresh credentials next time.
574
+ refresh = True
575
+ delay = min(self._backoff_s * 2 ** (attempt - 1), MAX_BACKOFF_S)
576
+ log.warning(
577
+ "%s: connection attempt %d/%d failed (%s); retrying in %.2fs",
578
+ self._endpoint, attempt, self._max_attempts, type(exc).__name__, delay,
579
+ )
580
+ self._sleep(delay)
581
+ continue
582
+ conn.client = client
583
+ return
584
+
585
+ @staticmethod
586
+ def _discard(client: Any) -> None:
587
+ # A connect that failed has usually torn itself down; closing it again waits.
588
+ if getattr(getattr(client, "eio", None), "state", "connected") == "disconnected":
589
+ return
590
+ try:
591
+ client.disconnect()
592
+ except Exception: # noqa: BLE001 - a half-open client being tidied; the connect error is the one that matters
593
+ log.debug("Socket.IO client tidy-up after a failed connect raised", exc_info=True)
594
+
595
+ def _catch_all(self, conn: _Conn) -> Callable[..., None]:
596
+ def handler(event: str, *args: Any) -> None:
597
+ conn.inbox.put((event, args[0] if len(args) == 1 else list(args)))
598
+
599
+ return handler
600
+
601
+ def _on_disconnect(self, conn: _Conn) -> Callable[..., None]:
602
+ def handler(*_: Any) -> None:
603
+ # Set before the sentinel, so a waiting turn that wakes sees why.
604
+ if conn.lost is None:
605
+ conn.lost = "the server closed the connection"
606
+ conn.inbox.put((_DISCONNECT, None))
607
+
608
+ return handler
609
+
610
+ @staticmethod
611
+ def _drain(conn: _Conn) -> int:
612
+ dropped = 0
613
+ while True:
614
+ try:
615
+ event, _ = conn.inbox.get_nowait()
616
+ except queue.Empty:
617
+ return dropped
618
+ if event != _DISCONNECT:
619
+ dropped += 1
620
+
621
+ # ------------------------------------------------------------------ #
622
+ # Reading a reply
623
+ # ------------------------------------------------------------------ #
624
+
625
+ def _await_reply(self, conn: _Conn, session_id: str) -> tuple[dict[str, Any], list[str]]:
626
+ """Classify events until the reply is complete: terminator, idle, error or timeout."""
627
+ deadline = time.monotonic() + self._timeout_s
628
+ merged: dict[str, Any] = {}
629
+ texts: list[str] = []
630
+ last = time.monotonic()
631
+ while True:
632
+ now = time.monotonic()
633
+ wait = deadline - now
634
+ if wait <= 0:
635
+ raise AgentCallError(
636
+ f"{self._endpoint} sent no complete reply for session {session_id!r} within "
637
+ f"{self._timeout_s:g}s; the turn was delivered and is not retried"
638
+ )
639
+ if self._idle_timeout_s is not None and texts:
640
+ quiet_left = self._idle_timeout_s - (now - last)
641
+ if quiet_left <= 0:
642
+ return merged, texts
643
+ wait = min(wait, quiet_left)
644
+ try:
645
+ event, payload = conn.inbox.get(timeout=wait)
646
+ except queue.Empty:
647
+ continue
648
+
649
+ if event == _DISCONNECT:
650
+ raise AgentCallError(
651
+ f"{self._endpoint} dropped the connection while session {session_id!r} "
652
+ "was waiting for a reply. The turn may have been delivered, so it was not "
653
+ "retried, and the session is lost."
654
+ )
655
+ kind = self._events.classify(event)
656
+ if kind == "ignorable":
657
+ self._note_unclassified(event)
658
+ continue
659
+ last = time.monotonic()
660
+ if kind == "error":
661
+ raise AgentCallError(
662
+ f"{self._endpoint} sent error event {event!r} for session {session_id!r}: "
663
+ f"{_snippet(payload)}"
664
+ )
665
+ if isinstance(payload, dict):
666
+ _merge(merged, payload)
667
+ if kind == "content":
668
+ piece = self._text_of(payload, event)
669
+ if piece is not None:
670
+ if self._content_mode == "replace":
671
+ texts[:] = [piece]
672
+ else:
673
+ texts.append(piece)
674
+ elif kind == "terminator":
675
+ if not texts:
676
+ raise AgentCallError(
677
+ f"{self._endpoint} sent terminator {event!r} for session {session_id!r} "
678
+ "with no content event before it; there is no reply to grade"
679
+ )
680
+ return merged, texts
681
+
682
+ def _text_of(self, payload: Any, event: str) -> str | None:
683
+ if isinstance(payload, str):
684
+ return payload
685
+ found = dig(payload, self._mapping["text"]) if isinstance(payload, dict) else MISSING
686
+ if found is MISSING:
687
+ return None
688
+ if not isinstance(found, str):
689
+ raise AgentCallError(
690
+ f"content event {event!r}: response_mapping.text path {self._mapping['text']!r} "
691
+ f"is a {type(found).__name__}, not text"
692
+ )
693
+ return found
694
+
695
+ def _note_unclassified(self, event: str) -> None:
696
+ with self._conns_lock:
697
+ first = event not in self._unclassified
698
+ self._unclassified.add(event)
699
+ if first:
700
+ log.warning(
701
+ "%s: ignoring event %r, which agent.events does not classify. If it carries "
702
+ "reply text or metadata, list it under content or metadata.",
703
+ self._endpoint, event,
704
+ )
705
+
706
+ def _map(
707
+ self, merged: dict[str, Any], texts: list[str], session_id: str, latency_ms: float
708
+ ) -> ChatResponse:
709
+ m = self._mapping
710
+ route, intent, session = dig(merged, m["route"]), dig(merged, m["intent"]), dig(merged, m["session"])
711
+ return ChatResponse(
712
+ reply="".join(texts),
713
+ conversation_id=session if isinstance(session, str) and session else session_id,
714
+ route=route if isinstance(route, str) and route else "unknown",
715
+ status=status_from(dig(merged, m["status"])),
716
+ intent=Intent(name=intent if isinstance(intent, str) and intent else "unknown"),
717
+ citations=[citation(i, c) for i, c in enumerate(self._list(merged, "citations"))],
718
+ retrieved=[chunk(i, c) for i, c in enumerate(self._list(merged, "retrieved"))],
719
+ tool_calls=[tool(i, t) for i, t in enumerate(self._list(merged, "tools"))],
720
+ latency_ms=round(latency_ms, 3),
721
+ )
722
+
723
+ def _list(self, merged: dict[str, Any], key: str) -> list[Any]:
724
+ """An optional list field: absent is empty, present-but-wrong is a contract break."""
725
+ value = dig(merged, self._mapping[key])
726
+ if value is MISSING or value is None:
727
+ return []
728
+ if not isinstance(value, list):
729
+ raise AgentCallError(
730
+ f"response_mapping.{key} path {self._mapping[key]!r} is a "
731
+ f"{type(value).__name__}, not a list"
732
+ )
733
+ return value
734
+
735
+
736
+ # --------------------------------------------------------------------------- #
737
+ # Helpers
738
+ # --------------------------------------------------------------------------- #
739
+
740
+
741
+ def _merge(into: dict[str, Any], payload: Mapping[str, Any]) -> None:
742
+ """Fold one event's payload into the reply's: lists extend, objects merge, scalars replace."""
743
+ for key, value in payload.items():
744
+ current = into.get(key)
745
+ if isinstance(current, list) and isinstance(value, list):
746
+ current.extend(value)
747
+ elif isinstance(current, dict) and isinstance(value, dict):
748
+ _merge(current, value)
749
+ else:
750
+ into[key] = list(value) if isinstance(value, list) else value
751
+
752
+
753
+ def _snippet(payload: Any) -> str:
754
+ text = " ".join(str(payload).split())
755
+ return text[:200] + ("..." if len(text) > 200 else "") or "(empty)"
756
+
757
+
758
+ def _events_from(raw: Any) -> Events:
759
+ if not isinstance(raw, Mapping):
760
+ raise ConfigError("agent.events is required: a mapping with at least `send` and `content`")
761
+ stray = sorted(set(raw) - {"send", "content", "metadata", "terminator", "errors"})
762
+ if stray:
763
+ raise ConfigError(
764
+ f"unknown agent.events key(s): {stray}; known: content, errors, metadata, send, terminator"
765
+ )
766
+ send = raw.get("send")
767
+ if not isinstance(send, str) or not send:
768
+ raise ConfigError("agent.events.send must be the event name a turn is emitted on")
769
+ content = _name_set(raw, "content")
770
+ if not content:
771
+ raise ConfigError("agent.events.content must name at least one event that carries reply text")
772
+ terminator = raw.get("terminator")
773
+ if terminator is not None and (not isinstance(terminator, str) or not terminator):
774
+ raise ConfigError(f"agent.events.terminator must be an event name, got {terminator!r}")
775
+ return Events(
776
+ send=send,
777
+ content=content,
778
+ metadata=_name_set(raw, "metadata"),
779
+ terminator=terminator,
780
+ errors=_name_set(raw, "errors"),
781
+ )
782
+
783
+
784
+ def _name_set(raw: Mapping[str, Any], key: str) -> frozenset[str]:
785
+ value = raw.get(key, [])
786
+ if isinstance(value, str):
787
+ value = [value]
788
+ if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value):
789
+ raise ConfigError(f"agent.events.{key} must be a list of event names, got {value!r}")
790
+ return frozenset(value)
791
+
792
+
793
+ def _auth_from(agent: Mapping[str, Any]) -> ConnectionAuth:
794
+ hook, headers = agent.get("auth"), agent.get("principal_headers")
795
+ if hook is not None and headers is not None:
796
+ raise ConfigError("give agent.auth (a hook) or agent.principal_headers (static), not both")
797
+ if hook is not None:
798
+ if not isinstance(hook, Mapping):
799
+ raise ConfigError(f"agent.auth must be a mapping, got {hook!r}")
800
+ return load_auth(hook)
801
+ if not isinstance(headers, Mapping):
802
+ raise ConfigError(
803
+ "agent.principal_headers is required for the socketio adapter: a mapping of "
804
+ "principal label -> {Header: value}. Use `default: {}` for an agent with no "
805
+ "authentication; a missing label never falls back to anonymous. "
806
+ "Or set agent.auth for a login hook."
807
+ )
808
+ for label, value in headers.items():
809
+ if not isinstance(value, Mapping):
810
+ raise ConfigError(f"agent.principal_headers.{label} must be a mapping, got {value!r}")
811
+ fields = agent.get("principal_auth")
812
+ if fields is not None and not isinstance(fields, Mapping):
813
+ raise ConfigError(f"agent.principal_auth must be a mapping, got {fields!r}")
814
+ return StaticAuth(headers, fields)
815
+
816
+
817
+ def _rate_limit_from(raw: Any) -> list[tuple[int, float]]:
818
+ if raw is None:
819
+ return []
820
+ if not isinstance(raw, list):
821
+ raise ConfigError(f"agent.rate_limit must be a list of {{requests, per_s}}, got {raw!r}")
822
+ windows: list[tuple[int, float]] = []
823
+ for entry in raw:
824
+ if not isinstance(entry, Mapping) or set(entry) != {"requests", "per_s"}:
825
+ raise ConfigError(f"agent.rate_limit entries need exactly requests and per_s, got {entry!r}")
826
+ count, per_s = entry["requests"], entry["per_s"]
827
+ if (
828
+ isinstance(count, bool) or not isinstance(count, int) or count < 1
829
+ or isinstance(per_s, bool) or not isinstance(per_s, (int, float)) or per_s <= 0
830
+ ): # fmt: skip
831
+ raise ConfigError(f"agent.rate_limit entry {entry!r}: requests >= 1 and per_s > 0")
832
+ windows.append((count, float(per_s)))
833
+ return windows
834
+
835
+
836
+ def _optional_mapping(agent: Mapping[str, Any], key: str) -> Mapping[str, Any] | None:
837
+ value = agent.get(key)
838
+ if value is not None and not isinstance(value, Mapping):
839
+ raise ConfigError(f"agent.{key} must be a mapping, got {value!r}")
840
+ return value
841
+
842
+
843
+ def _text(agent: Mapping[str, Any], key: str, default: str) -> str:
844
+ value = agent.get(key, default)
845
+ if not isinstance(value, str) or not value:
846
+ raise ConfigError(f"agent.{key} must be non-empty text, got {value!r}")
847
+ return value
848
+
849
+
850
+ def _str_list(agent: Mapping[str, Any], key: str) -> list[str]:
851
+ value = agent.get(key) or []
852
+ if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
853
+ raise ConfigError(f"agent.{key} must be a list of strings, got {value!r}")
854
+ return value
855
+
856
+
857
+ def _number(agent: Mapping[str, Any], key: str, default: float | None) -> float | None:
858
+ value = agent.get(key, default)
859
+ if value is None:
860
+ return None
861
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
862
+ raise ConfigError(f"agent.{key} must be a number, got {value!r}")
863
+ return float(value)
864
+
865
+
866
+ def _float(agent: Mapping[str, Any], key: str, default: float) -> float:
867
+ """A number with a default. A present value is checked as it is: 0 is not "unset"."""
868
+ value = _number(agent, key, default)
869
+ assert value is not None # the default is a float, so None cannot come back
870
+ return value
871
+
872
+
873
+ __all__ = [
874
+ "INSTALL_HINT",
875
+ "CachedTokenAuth",
876
+ "ConnectionAuth",
877
+ "Credentials",
878
+ "Events",
879
+ "RateLimiter",
880
+ "SocketIOAdapter",
881
+ "StaticAuth",
882
+ "Token",
883
+ "jwt_expiry",
884
+ "load_auth",
885
+ ]