askback 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. askback/__init__.py +29 -0
  2. askback/__main__.py +8 -0
  3. askback/adapters/__init__.py +11 -0
  4. askback/adapters/base.py +189 -0
  5. askback/adapters/factory.py +176 -0
  6. askback/adapters/http.py +443 -0
  7. askback/adapters/in_process.py +135 -0
  8. askback/adapters/payload.py +207 -0
  9. askback/adapters/scripted.py +117 -0
  10. askback/adapters/socketio_adapter.py +885 -0
  11. askback/api/__init__.py +18 -0
  12. askback/api/corpus_store.py +85 -0
  13. askback/api/dashboard.py +444 -0
  14. askback/api/server.py +200 -0
  15. askback/api/state.py +82 -0
  16. askback/api/static/app.css +423 -0
  17. askback/api/static/app.js +98 -0
  18. askback/api/static/core.js +201 -0
  19. askback/api/static/index.html +175 -0
  20. askback/api/static/panels.js +651 -0
  21. askback/cli.py +618 -0
  22. askback/core/__init__.py +58 -0
  23. askback/core/config.py +224 -0
  24. askback/core/contracts.py +201 -0
  25. askback/core/errors.py +35 -0
  26. askback/core/logging.py +57 -0
  27. askback/core/registry.py +73 -0
  28. askback/core/text.py +151 -0
  29. askback/datasets/__init__.py +14 -0
  30. askback/datasets/loader.py +268 -0
  31. askback/datasets/schema.py +525 -0
  32. askback/demo/__init__.py +42 -0
  33. askback/demo/agent.py +195 -0
  34. askback/demo/suites/demo.yaml +100 -0
  35. askback/evaluation/__init__.py +70 -0
  36. askback/evaluation/base.py +214 -0
  37. askback/evaluation/calibration.py +405 -0
  38. askback/evaluation/citation_eval.py +580 -0
  39. askback/evaluation/clarification.py +477 -0
  40. askback/evaluation/engine.py +193 -0
  41. askback/evaluation/escalation_eval.py +109 -0
  42. askback/evaluation/gates.py +351 -0
  43. askback/evaluation/judge_cache.py +94 -0
  44. askback/evaluation/llm_judge.py +463 -0
  45. askback/evaluation/patterns.py +48 -0
  46. askback/evaluation/pii_eval.py +252 -0
  47. askback/evaluation/rag_eval.py +168 -0
  48. askback/evaluation/regression.py +438 -0
  49. askback/evaluation/results.py +588 -0
  50. askback/evaluation/rule_based.py +198 -0
  51. askback/evaluation/safety_eval.py +170 -0
  52. askback/providers/__init__.py +10 -0
  53. askback/providers/base.py +157 -0
  54. askback/providers/openai_compat.py +340 -0
  55. askback/py.typed +0 -0
  56. askback/reporting/__init__.py +77 -0
  57. askback/reporting/excel_report.py +117 -0
  58. askback/reporting/html_report.py +283 -0
  59. askback/reporting/json_report.py +27 -0
  60. askback/reporting/markdown_report.py +95 -0
  61. askback/reporting/redaction.py +185 -0
  62. askback/runner/__init__.py +5 -0
  63. askback/runner/responders.py +279 -0
  64. askback/runner/test_runner.py +424 -0
  65. askback-0.1.0.dist-info/METADATA +504 -0
  66. askback-0.1.0.dist-info/RECORD +71 -0
  67. askback-0.1.0.dist-info/WHEEL +5 -0
  68. askback-0.1.0.dist-info/entry_points.txt +2 -0
  69. askback-0.1.0.dist-info/licenses/LICENSE +176 -0
  70. askback-0.1.0.dist-info/licenses/NOTICE +15 -0
  71. askback-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,207 @@
1
+ """Request templates and response mappings, shared by the adapters that speak JSON.
2
+
3
+ An agent's payload is its own. What differs between agents is how to ask (a request
4
+ template with `{message}` and `{session_id}`), and where the answer lives (dotted
5
+ response paths), so a payload change is a YAML edit and not a code change. This
6
+ module owns those two things and imports no transport, so an adapter over HTTP and
7
+ one over Socket.IO read an agent's reply the same way.
8
+
9
+ Honest reporting lives here too. A field the agent does not return stays empty, so
10
+ the evaluators that need it call the case unmeasurable, and a status that is absent
11
+ or not one we know becomes `ResponseStatus.UNKNOWN`, never `ANSWERED`.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import re
17
+ from collections.abc import Mapping
18
+ from typing import Any
19
+
20
+ from ..core.contracts import Citation, ResponseStatus, RetrievedChunk, ToolCall
21
+ from ..core.errors import AgentCallError, ConfigError
22
+
23
+ #: Dotted paths into the response. The defaults are the key names of this repo's
24
+ #: own `ChatResponse.to_dict()`; `text` is the only mandatory one. Nothing here
25
+ #: works against a particular agent unconfigured - the simulator's own API takes
26
+ #: `message` and `conversation_id`, not the request template above, and its
27
+ #: identity in the body. tests/test_acceptance.py carries the config that does.
28
+ DEFAULT_RESPONSE_MAPPING: Mapping[str, str] = {
29
+ "text": "reply",
30
+ "route": "route",
31
+ "status": "status",
32
+ "intent": "intent.name",
33
+ "citations": "citations",
34
+ "retrieved": "retrieved",
35
+ "tools": "tool_calls",
36
+ "session": "conversation_id",
37
+ }
38
+
39
+ PLACEHOLDER = re.compile(r"\{(\w+)\}")
40
+ PLACEHOLDERS = frozenset({"message", "session_id"})
41
+ MISSING = object()
42
+
43
+
44
+ def validated_template(template: Mapping[str, Any]) -> Mapping[str, Any]:
45
+ seen: set[str] = set()
46
+
47
+ def walk(node: Any, where: str) -> None:
48
+ if isinstance(node, str):
49
+ for name in PLACEHOLDER.findall(node):
50
+ if name not in PLACEHOLDERS:
51
+ raise ConfigError(
52
+ f"agent.request_template{where} uses {{{name}}}; only "
53
+ f"{sorted(PLACEHOLDERS)} are filled in"
54
+ )
55
+ seen.add(name)
56
+ elif isinstance(node, Mapping):
57
+ for key, value in node.items():
58
+ walk(value, f"{where}.{key}")
59
+ elif isinstance(node, list):
60
+ for i, value in enumerate(node):
61
+ walk(value, f"{where}[{i}]")
62
+
63
+ walk(template, "")
64
+ if "message" not in seen:
65
+ raise ConfigError(
66
+ "agent.request_template never uses {message}: every case would send the "
67
+ "same turn and the run would measure nothing"
68
+ )
69
+ return template
70
+
71
+
72
+ def fill(node: Any, values: Mapping[str, str]) -> Any:
73
+ """Substitute placeholders, in one pass over the template only.
74
+
75
+ Not `str.format`: the template is JSON-ish and full of braces. Not repeated
76
+ replacement either: a message that itself contains "{session_id}" must reach
77
+ the agent as typed.
78
+ """
79
+ if isinstance(node, str):
80
+ return PLACEHOLDER.sub(lambda m: values.get(m.group(1), m.group(0)), node)
81
+ if isinstance(node, Mapping):
82
+ return {key: fill(value, values) for key, value in node.items()}
83
+ if isinstance(node, list):
84
+ return [fill(value, values) for value in node]
85
+ return node
86
+
87
+
88
+ def validated_headers(headers: Mapping[str, Any], where: str) -> dict[str, str]:
89
+ out: dict[str, str] = {}
90
+ for name, value in headers.items():
91
+ # An unset ${TOKEN} expands to "". Refuse it here, by name, rather than
92
+ # send an unauthenticated request and read the 401 as an agent failure.
93
+ # The value is never echoed: it may be a credential.
94
+ if not isinstance(name, str) or not isinstance(value, str) or not value:
95
+ raise ConfigError(
96
+ f"{where}.{name} must be a non-empty string (is its environment variable set?)"
97
+ )
98
+ out[name] = value
99
+ return out
100
+
101
+
102
+ def validated_mapping(overrides: Mapping[str, str] | None) -> dict[str, str]:
103
+ mapping = dict(DEFAULT_RESPONSE_MAPPING)
104
+ for key, path in (overrides or {}).items():
105
+ if key not in DEFAULT_RESPONSE_MAPPING:
106
+ raise ConfigError(
107
+ f"unknown agent.response_mapping key {key!r}; "
108
+ f"valid keys: {', '.join(DEFAULT_RESPONSE_MAPPING)}"
109
+ )
110
+ if not isinstance(path, str) or not path:
111
+ raise ConfigError(f"agent.response_mapping.{key} must be a dotted path, got {path!r}")
112
+ mapping[key] = path
113
+ return mapping
114
+
115
+
116
+ def dig(payload: Any, path: str) -> Any:
117
+ """Follow a dotted path through dicts and lists (`choices.0.message.content`)."""
118
+ node = payload
119
+ for part in path.split("."):
120
+ if isinstance(node, dict) and part in node:
121
+ node = node[part]
122
+ elif isinstance(node, list) and part.isdigit() and int(part) < len(node):
123
+ node = node[int(part)]
124
+ else:
125
+ return MISSING
126
+ return node
127
+
128
+
129
+ def status_from(raw: Any) -> ResponseStatus:
130
+ """Map the agent's status string, or say we cannot. Never a success default."""
131
+ if isinstance(raw, str):
132
+ try:
133
+ return ResponseStatus(raw.strip().lower())
134
+ except ValueError:
135
+ pass
136
+ return ResponseStatus.UNKNOWN
137
+
138
+
139
+ def citation(index: int, raw: Any) -> Citation:
140
+ item = item_of(raw, "citations", index, required="doc_id")
141
+ return Citation(
142
+ doc_id=str(item["doc_id"]),
143
+ chunk_id=str(item.get("chunk_id", "")),
144
+ snippet=str(item.get("snippet", "")),
145
+ title=str(item.get("title", "")),
146
+ source=str(item.get("source", "")),
147
+ )
148
+
149
+
150
+ def chunk(index: int, raw: Any) -> RetrievedChunk:
151
+ item = item_of(raw, "retrieved", index, required="doc_id")
152
+ if not isinstance(item.get("text"), str):
153
+ raise AgentCallError(f"retrieved[{index}] has no text; its keys are {sorted(item)}")
154
+ score = item.get("score", 0.0)
155
+ # Carried through, not dropped: evaluators read it (the tenant a chunk
156
+ # belongs to), and a check whose data went missing here passes vacuously.
157
+ metadata = item.get("metadata")
158
+ if metadata is not None and not isinstance(metadata, dict):
159
+ raise AgentCallError(
160
+ f"retrieved[{index}].metadata is a {type(metadata).__name__}, not an object"
161
+ )
162
+ return RetrievedChunk(
163
+ doc_id=str(item["doc_id"]),
164
+ chunk_id=str(item.get("chunk_id", "")),
165
+ text=item["text"],
166
+ score=float(score) if isinstance(score, (int, float)) else 0.0,
167
+ title=str(item.get("title", "")),
168
+ source=str(item.get("source", "")),
169
+ metadata=dict(metadata or {}),
170
+ )
171
+
172
+
173
+ def tool(index: int, raw: Any) -> ToolCall:
174
+ if isinstance(raw, str):
175
+ return ToolCall(name=raw)
176
+ item = item_of(raw, "tools", index, required="name")
177
+ arguments = item.get("arguments", {})
178
+ return ToolCall(
179
+ name=str(item["name"]),
180
+ arguments=arguments if isinstance(arguments, dict) else {},
181
+ output=item.get("output"),
182
+ ok=bool(item.get("ok", True)),
183
+ error=str(item["error"]) if item.get("error") else None,
184
+ )
185
+
186
+
187
+ def item_of(raw: Any, field: str, index: int, *, required: str) -> dict[str, Any]:
188
+ if not isinstance(raw, dict) or required not in raw:
189
+ got = f"keys {sorted(raw)}" if isinstance(raw, dict) else type(raw).__name__
190
+ raise AgentCallError(f"{field}[{index}] needs a {required!r}; got {got}")
191
+ return raw
192
+
193
+
194
+ __all__ = [
195
+ "DEFAULT_RESPONSE_MAPPING",
196
+ "MISSING",
197
+ "PLACEHOLDERS",
198
+ "chunk",
199
+ "citation",
200
+ "dig",
201
+ "fill",
202
+ "status_from",
203
+ "tool",
204
+ "validated_headers",
205
+ "validated_mapping",
206
+ "validated_template",
207
+ ]
@@ -0,0 +1,117 @@
1
+ """The scripted adapter: canned answers behind the adapter seam.
2
+
3
+ This is how a user of the library tests their own evaluators, gates and reports
4
+ without standing up an agent. The response for each message is decided by the
5
+ test, so a failing check points at the evaluator rather than at a model's mood.
6
+ It is a real implementation of `AgentAdapter`, not a mock: the runner cannot tell
7
+ it from any other adapter.
8
+
9
+ It imports nothing from the simulator on purpose. It has to keep working after
10
+ the simulator is split out into its own package.
11
+
12
+ Unlike `InProcessAdapter` it does not resolve principals. The label is recorded
13
+ verbatim on `.calls`; turning a label into an identity is what a real adapter
14
+ does, and a script has no identity to give.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import copy
20
+ import threading
21
+ from collections.abc import Callable, Mapping
22
+
23
+ from ..core.contracts import ChatResponse
24
+ from ..core.errors import ConfigError
25
+ from .base import AdapterLifecycle
26
+
27
+ #: (message, prior user messages in this session, oldest first) -> response.
28
+ Responder = Callable[[str, list[str]], ChatResponse]
29
+
30
+
31
+ class ScriptedAdapter(AdapterLifecycle):
32
+ """`AgentAdapter` that answers from a script.
33
+
34
+ Give it either `replies` (exact message -> response) or a `responder`
35
+ (computes the response, and sees the session's earlier messages), never both.
36
+ `raises` is checked first, so a message can be forced to fail whichever mode
37
+ is in use. A message nothing matches gets `default`; with no `default` it
38
+ raises, because inventing a reply would hand an evaluator something the
39
+ script never said.
40
+
41
+ A stored reply is handed out as a deep copy. Evaluators receive the response
42
+ object itself, so one that mutated it would otherwise rewrite the script for
43
+ every later call. A `responder` builds its reply per call and is trusted to
44
+ return a fresh one.
45
+ """
46
+
47
+ def __init__(
48
+ self,
49
+ *,
50
+ replies: Mapping[str, ChatResponse] | None = None,
51
+ responder: Responder | None = None,
52
+ default: ChatResponse | None = None,
53
+ raises: Mapping[str, Exception] | None = None,
54
+ ) -> None:
55
+ super().__init__()
56
+ if (replies is None) == (responder is None):
57
+ raise ConfigError(
58
+ "ScriptedAdapter needs exactly one of `replies` or `responder`."
59
+ )
60
+ # Copied so a test mutating its dicts mid-run cannot rewrite the script.
61
+ self._replies = dict(replies) if replies is not None else None
62
+ self._responder = responder
63
+ self._default = default
64
+ self._raises = dict(raises or {})
65
+
66
+ #: Every call, in arrival order, as (message, session_id, principal).
67
+ self.calls: list[tuple[str, str, str]] = []
68
+ self._history: dict[str, list[str]] = {}
69
+ # Guards `calls` and `_history`, which workers share. The responder runs
70
+ # outside it so a slow one does not serialise the sessions.
71
+ self._state_lock = threading.Lock()
72
+
73
+ def send(
74
+ self, message: str, *, session_id: str, principal: str = "default"
75
+ ) -> ChatResponse:
76
+ """Record the call, then answer it from the script.
77
+
78
+ A forced failure still counts as a call, but not as a turn the session
79
+ remembers: the system never answered it, so later messages do not see it.
80
+ """
81
+ self._require_open("send")
82
+ with self._state_lock:
83
+ self.calls.append((message, session_id, principal))
84
+ prior = list(self._history.get(session_id, ()))
85
+
86
+ if message in self._raises:
87
+ raise self._raises[message]
88
+
89
+ response = self._respond(message, prior)
90
+ with self._state_lock:
91
+ self._history.setdefault(session_id, []).append(message)
92
+ return response
93
+
94
+ def describe(self) -> dict[str, str]:
95
+ """Provenance for the run report.
96
+
97
+ Only the adapter name. A script has no model, retriever or prompt, and
98
+ claiming a version for one would be provenance nobody can check.
99
+ """
100
+ self._require_open("describe")
101
+ return {"adapter": "scripted"}
102
+
103
+ def _respond(self, message: str, prior: list[str]) -> ChatResponse:
104
+ if self._responder is not None:
105
+ return self._responder(message, prior)
106
+ assert self._replies is not None # the constructor guarantees one mode
107
+ if message in self._replies:
108
+ return copy.deepcopy(self._replies[message])
109
+ if self._default is not None:
110
+ return copy.deepcopy(self._default)
111
+ raise ConfigError(
112
+ f"ScriptedAdapter has no reply for {message!r} and no default; "
113
+ "script it in `replies` or pass `default`."
114
+ )
115
+
116
+
117
+ __all__ = ["Responder", "ScriptedAdapter"]