askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Request templates and response mappings, shared by the adapters that speak JSON.
|
|
2
|
+
|
|
3
|
+
An agent's payload is its own. What differs between agents is how to ask (a request
|
|
4
|
+
template with `{message}` and `{session_id}`), and where the answer lives (dotted
|
|
5
|
+
response paths), so a payload change is a YAML edit and not a code change. This
|
|
6
|
+
module owns those two things and imports no transport, so an adapter over HTTP and
|
|
7
|
+
one over Socket.IO read an agent's reply the same way.
|
|
8
|
+
|
|
9
|
+
Honest reporting lives here too. A field the agent does not return stays empty, so
|
|
10
|
+
the evaluators that need it call the case unmeasurable, and a status that is absent
|
|
11
|
+
or not one we know becomes `ResponseStatus.UNKNOWN`, never `ANSWERED`.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from ..core.contracts import Citation, ResponseStatus, RetrievedChunk, ToolCall
|
|
21
|
+
from ..core.errors import AgentCallError, ConfigError
|
|
22
|
+
|
|
23
|
+
#: Dotted paths into the response. The defaults are the key names of this repo's
|
|
24
|
+
#: own `ChatResponse.to_dict()`; `text` is the only mandatory one. Nothing here
|
|
25
|
+
#: works against a particular agent unconfigured - the simulator's own API takes
|
|
26
|
+
#: `message` and `conversation_id`, not the request template above, and its
|
|
27
|
+
#: identity in the body. tests/test_acceptance.py carries the config that does.
|
|
28
|
+
DEFAULT_RESPONSE_MAPPING: Mapping[str, str] = {
|
|
29
|
+
"text": "reply",
|
|
30
|
+
"route": "route",
|
|
31
|
+
"status": "status",
|
|
32
|
+
"intent": "intent.name",
|
|
33
|
+
"citations": "citations",
|
|
34
|
+
"retrieved": "retrieved",
|
|
35
|
+
"tools": "tool_calls",
|
|
36
|
+
"session": "conversation_id",
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
PLACEHOLDER = re.compile(r"\{(\w+)\}")
|
|
40
|
+
PLACEHOLDERS = frozenset({"message", "session_id"})
|
|
41
|
+
MISSING = object()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def validated_template(template: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
45
|
+
seen: set[str] = set()
|
|
46
|
+
|
|
47
|
+
def walk(node: Any, where: str) -> None:
|
|
48
|
+
if isinstance(node, str):
|
|
49
|
+
for name in PLACEHOLDER.findall(node):
|
|
50
|
+
if name not in PLACEHOLDERS:
|
|
51
|
+
raise ConfigError(
|
|
52
|
+
f"agent.request_template{where} uses {{{name}}}; only "
|
|
53
|
+
f"{sorted(PLACEHOLDERS)} are filled in"
|
|
54
|
+
)
|
|
55
|
+
seen.add(name)
|
|
56
|
+
elif isinstance(node, Mapping):
|
|
57
|
+
for key, value in node.items():
|
|
58
|
+
walk(value, f"{where}.{key}")
|
|
59
|
+
elif isinstance(node, list):
|
|
60
|
+
for i, value in enumerate(node):
|
|
61
|
+
walk(value, f"{where}[{i}]")
|
|
62
|
+
|
|
63
|
+
walk(template, "")
|
|
64
|
+
if "message" not in seen:
|
|
65
|
+
raise ConfigError(
|
|
66
|
+
"agent.request_template never uses {message}: every case would send the "
|
|
67
|
+
"same turn and the run would measure nothing"
|
|
68
|
+
)
|
|
69
|
+
return template
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def fill(node: Any, values: Mapping[str, str]) -> Any:
|
|
73
|
+
"""Substitute placeholders, in one pass over the template only.
|
|
74
|
+
|
|
75
|
+
Not `str.format`: the template is JSON-ish and full of braces. Not repeated
|
|
76
|
+
replacement either: a message that itself contains "{session_id}" must reach
|
|
77
|
+
the agent as typed.
|
|
78
|
+
"""
|
|
79
|
+
if isinstance(node, str):
|
|
80
|
+
return PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), node)
|
|
81
|
+
if isinstance(node, Mapping):
|
|
82
|
+
return {key: fill(value, values) for key, value in node.items()}
|
|
83
|
+
if isinstance(node, list):
|
|
84
|
+
return [fill(value, values) for value in node]
|
|
85
|
+
return node
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def validated_headers(headers: Mapping[str, Any], where: str) -> dict[str, str]:
|
|
89
|
+
out: dict[str, str] = {}
|
|
90
|
+
for name, value in headers.items():
|
|
91
|
+
# An unset ${TOKEN} expands to "". Refuse it here, by name, rather than
|
|
92
|
+
# send an unauthenticated request and read the 401 as an agent failure.
|
|
93
|
+
# The value is never echoed: it may be a credential.
|
|
94
|
+
if not isinstance(name, str) or not isinstance(value, str) or not value:
|
|
95
|
+
raise ConfigError(
|
|
96
|
+
f"{where}.{name} must be a non-empty string (is its environment variable set?)"
|
|
97
|
+
)
|
|
98
|
+
out[name] = value
|
|
99
|
+
return out
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def validated_mapping(overrides: Mapping[str, str] | None) -> dict[str, str]:
|
|
103
|
+
mapping = dict(DEFAULT_RESPONSE_MAPPING)
|
|
104
|
+
for key, path in (overrides or {}).items():
|
|
105
|
+
if key not in DEFAULT_RESPONSE_MAPPING:
|
|
106
|
+
raise ConfigError(
|
|
107
|
+
f"unknown agent.response_mapping key {key!r}; "
|
|
108
|
+
f"valid keys: {', '.join(DEFAULT_RESPONSE_MAPPING)}"
|
|
109
|
+
)
|
|
110
|
+
if not isinstance(path, str) or not path:
|
|
111
|
+
raise ConfigError(f"agent.response_mapping.{key} must be a dotted path, got {path!r}")
|
|
112
|
+
mapping[key] = path
|
|
113
|
+
return mapping
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def dig(payload: Any, path: str) -> Any:
|
|
117
|
+
"""Follow a dotted path through dicts and lists (`choices.0.message.content`)."""
|
|
118
|
+
node = payload
|
|
119
|
+
for part in path.split("."):
|
|
120
|
+
if isinstance(node, dict) and part in node:
|
|
121
|
+
node = node[part]
|
|
122
|
+
elif isinstance(node, list) and part.isdigit() and int(part) < len(node):
|
|
123
|
+
node = node[int(part)]
|
|
124
|
+
else:
|
|
125
|
+
return MISSING
|
|
126
|
+
return node
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def status_from(raw: Any) -> ResponseStatus:
|
|
130
|
+
"""Map the agent's status string, or say we cannot. Never a success default."""
|
|
131
|
+
if isinstance(raw, str):
|
|
132
|
+
try:
|
|
133
|
+
return ResponseStatus(raw.strip().lower())
|
|
134
|
+
except ValueError:
|
|
135
|
+
pass
|
|
136
|
+
return ResponseStatus.UNKNOWN
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def citation(index: int, raw: Any) -> Citation:
|
|
140
|
+
item = item_of(raw, "citations", index, required="doc_id")
|
|
141
|
+
return Citation(
|
|
142
|
+
doc_id=str(item["doc_id"]),
|
|
143
|
+
chunk_id=str(item.get("chunk_id", "")),
|
|
144
|
+
snippet=str(item.get("snippet", "")),
|
|
145
|
+
title=str(item.get("title", "")),
|
|
146
|
+
source=str(item.get("source", "")),
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def chunk(index: int, raw: Any) -> RetrievedChunk:
|
|
151
|
+
item = item_of(raw, "retrieved", index, required="doc_id")
|
|
152
|
+
if not isinstance(item.get("text"), str):
|
|
153
|
+
raise AgentCallError(f"retrieved[{index}] has no text; its keys are {sorted(item)}")
|
|
154
|
+
score = item.get("score", 0.0)
|
|
155
|
+
# Carried through, not dropped: evaluators read it (the tenant a chunk
|
|
156
|
+
# belongs to), and a check whose data went missing here passes vacuously.
|
|
157
|
+
metadata = item.get("metadata")
|
|
158
|
+
if metadata is not None and not isinstance(metadata, dict):
|
|
159
|
+
raise AgentCallError(
|
|
160
|
+
f"retrieved[{index}].metadata is a {type(metadata).__name__}, not an object"
|
|
161
|
+
)
|
|
162
|
+
return RetrievedChunk(
|
|
163
|
+
doc_id=str(item["doc_id"]),
|
|
164
|
+
chunk_id=str(item.get("chunk_id", "")),
|
|
165
|
+
text=item["text"],
|
|
166
|
+
score=float(score) if isinstance(score, (int, float)) else 0.0,
|
|
167
|
+
title=str(item.get("title", "")),
|
|
168
|
+
source=str(item.get("source", "")),
|
|
169
|
+
metadata=dict(metadata or {}),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def tool(index: int, raw: Any) -> ToolCall:
|
|
174
|
+
if isinstance(raw, str):
|
|
175
|
+
return ToolCall(name=raw)
|
|
176
|
+
item = item_of(raw, "tools", index, required="name")
|
|
177
|
+
arguments = item.get("arguments", {})
|
|
178
|
+
return ToolCall(
|
|
179
|
+
name=str(item["name"]),
|
|
180
|
+
arguments=arguments if isinstance(arguments, dict) else {},
|
|
181
|
+
output=item.get("output"),
|
|
182
|
+
ok=bool(item.get("ok", True)),
|
|
183
|
+
error=str(item["error"]) if item.get("error") else None,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def item_of(raw: Any, field: str, index: int, *, required: str) -> dict[str, Any]:
|
|
188
|
+
if not isinstance(raw, dict) or required not in raw:
|
|
189
|
+
got = f"keys {sorted(raw)}" if isinstance(raw, dict) else type(raw).__name__
|
|
190
|
+
raise AgentCallError(f"{field}[{index}] needs a {required!r}; got {got}")
|
|
191
|
+
return raw
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
__all__ = [
|
|
195
|
+
"DEFAULT_RESPONSE_MAPPING",
|
|
196
|
+
"MISSING",
|
|
197
|
+
"PLACEHOLDERS",
|
|
198
|
+
"chunk",
|
|
199
|
+
"citation",
|
|
200
|
+
"dig",
|
|
201
|
+
"fill",
|
|
202
|
+
"status_from",
|
|
203
|
+
"tool",
|
|
204
|
+
"validated_headers",
|
|
205
|
+
"validated_mapping",
|
|
206
|
+
"validated_template",
|
|
207
|
+
]
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""The scripted adapter: canned answers behind the adapter seam.
|
|
2
|
+
|
|
3
|
+
This is how a user of the library tests their own evaluators, gates and reports
|
|
4
|
+
without standing up an agent. The response for each message is decided by the
|
|
5
|
+
test, so a failing check points at the evaluator rather than at a model's mood.
|
|
6
|
+
It is a real implementation of `AgentAdapter`, not a mock: the runner cannot tell
|
|
7
|
+
it from any other adapter.
|
|
8
|
+
|
|
9
|
+
It imports nothing from the simulator on purpose. It has to keep working after
|
|
10
|
+
the simulator is split out into its own package.
|
|
11
|
+
|
|
12
|
+
Unlike `InProcessAdapter` it does not resolve principals. The label is recorded
|
|
13
|
+
verbatim on `.calls`; turning a label into an identity is what a real adapter
|
|
14
|
+
does, and a script has no identity to give.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import copy
|
|
20
|
+
import threading
|
|
21
|
+
from collections.abc import Callable, Mapping
|
|
22
|
+
|
|
23
|
+
from ..core.contracts import ChatResponse
|
|
24
|
+
from ..core.errors import ConfigError
|
|
25
|
+
from .base import AdapterLifecycle
|
|
26
|
+
|
|
27
|
+
#: (message, prior user messages in this session, oldest first) -> response.
|
|
28
|
+
Responder = Callable[[str, list[str]], ChatResponse]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ScriptedAdapter(AdapterLifecycle):
|
|
32
|
+
"""`AgentAdapter` that answers from a script.
|
|
33
|
+
|
|
34
|
+
Give it either `replies` (exact message -> response) or a `responder`
|
|
35
|
+
(computes the response, and sees the session's earlier messages), never both.
|
|
36
|
+
`raises` is checked first, so a message can be forced to fail whichever mode
|
|
37
|
+
is in use. A message nothing matches gets `default`; with no `default` it
|
|
38
|
+
raises, because inventing a reply would hand an evaluator something the
|
|
39
|
+
script never said.
|
|
40
|
+
|
|
41
|
+
A stored reply is handed out as a deep copy. Evaluators receive the response
|
|
42
|
+
object itself, so one that mutated it would otherwise rewrite the script for
|
|
43
|
+
every later call. A `responder` builds its reply per call and is trusted to
|
|
44
|
+
return a fresh one.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
*,
|
|
50
|
+
replies: Mapping[str, ChatResponse] | None = None,
|
|
51
|
+
responder: Responder | None = None,
|
|
52
|
+
default: ChatResponse | None = None,
|
|
53
|
+
raises: Mapping[str, Exception] | None = None,
|
|
54
|
+
) -> None:
|
|
55
|
+
super().__init__()
|
|
56
|
+
if (replies is None) == (responder is None):
|
|
57
|
+
raise ConfigError(
|
|
58
|
+
"ScriptedAdapter needs exactly one of `replies` or `responder`."
|
|
59
|
+
)
|
|
60
|
+
# Copied so a test mutating its dicts mid-run cannot rewrite the script.
|
|
61
|
+
self._replies = dict(replies) if replies is not None else None
|
|
62
|
+
self._responder = responder
|
|
63
|
+
self._default = default
|
|
64
|
+
self._raises = dict(raises or {})
|
|
65
|
+
|
|
66
|
+
#: Every call, in arrival order, as (message, session_id, principal).
|
|
67
|
+
self.calls: list[tuple[str, str, str]] = []
|
|
68
|
+
self._history: dict[str, list[str]] = {}
|
|
69
|
+
# Guards `calls` and `_history`, which workers share. The responder runs
|
|
70
|
+
# outside it so a slow one does not serialise the sessions.
|
|
71
|
+
self._state_lock = threading.Lock()
|
|
72
|
+
|
|
73
|
+
def send(
|
|
74
|
+
self, message: str, *, session_id: str, principal: str = "default"
|
|
75
|
+
) -> ChatResponse:
|
|
76
|
+
"""Record the call, then answer it from the script.
|
|
77
|
+
|
|
78
|
+
A forced failure still counts as a call, but not as a turn the session
|
|
79
|
+
remembers: the system never answered it, so later messages do not see it.
|
|
80
|
+
"""
|
|
81
|
+
self._require_open("send")
|
|
82
|
+
with self._state_lock:
|
|
83
|
+
self.calls.append((message, session_id, principal))
|
|
84
|
+
prior = list(self._history.get(session_id, ()))
|
|
85
|
+
|
|
86
|
+
if message in self._raises:
|
|
87
|
+
raise self._raises[message]
|
|
88
|
+
|
|
89
|
+
response = self._respond(message, prior)
|
|
90
|
+
with self._state_lock:
|
|
91
|
+
self._history.setdefault(session_id, []).append(message)
|
|
92
|
+
return response
|
|
93
|
+
|
|
94
|
+
def describe(self) -> dict[str, str]:
|
|
95
|
+
"""Provenance for the run report.
|
|
96
|
+
|
|
97
|
+
Only the adapter name. A script has no model, retriever or prompt, and
|
|
98
|
+
claiming a version for one would be provenance nobody can check.
|
|
99
|
+
"""
|
|
100
|
+
self._require_open("describe")
|
|
101
|
+
return {"adapter": "scripted"}
|
|
102
|
+
|
|
103
|
+
def _respond(self, message: str, prior: list[str]) -> ChatResponse:
|
|
104
|
+
if self._responder is not None:
|
|
105
|
+
return self._responder(message, prior)
|
|
106
|
+
assert self._replies is not None # the constructor guarantees one mode
|
|
107
|
+
if message in self._replies:
|
|
108
|
+
return copy.deepcopy(self._replies[message])
|
|
109
|
+
if self._default is not None:
|
|
110
|
+
return copy.deepcopy(self._default)
|
|
111
|
+
raise ConfigError(
|
|
112
|
+
f"ScriptedAdapter has no reply for {message!r} and no default; "
|
|
113
|
+
"script it in `replies` or pass `default`."
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
__all__ = ["Responder", "ScriptedAdapter"]
|