askback 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. askback/__init__.py +29 -0
  2. askback/__main__.py +8 -0
  3. askback/adapters/__init__.py +11 -0
  4. askback/adapters/base.py +189 -0
  5. askback/adapters/factory.py +176 -0
  6. askback/adapters/http.py +443 -0
  7. askback/adapters/in_process.py +135 -0
  8. askback/adapters/payload.py +207 -0
  9. askback/adapters/scripted.py +117 -0
  10. askback/adapters/socketio_adapter.py +885 -0
  11. askback/api/__init__.py +18 -0
  12. askback/api/corpus_store.py +85 -0
  13. askback/api/dashboard.py +444 -0
  14. askback/api/server.py +200 -0
  15. askback/api/state.py +82 -0
  16. askback/api/static/app.css +423 -0
  17. askback/api/static/app.js +98 -0
  18. askback/api/static/core.js +201 -0
  19. askback/api/static/index.html +175 -0
  20. askback/api/static/panels.js +651 -0
  21. askback/cli.py +618 -0
  22. askback/core/__init__.py +58 -0
  23. askback/core/config.py +224 -0
  24. askback/core/contracts.py +201 -0
  25. askback/core/errors.py +35 -0
  26. askback/core/logging.py +57 -0
  27. askback/core/registry.py +73 -0
  28. askback/core/text.py +151 -0
  29. askback/datasets/__init__.py +14 -0
  30. askback/datasets/loader.py +268 -0
  31. askback/datasets/schema.py +525 -0
  32. askback/demo/__init__.py +42 -0
  33. askback/demo/agent.py +195 -0
  34. askback/demo/suites/demo.yaml +100 -0
  35. askback/evaluation/__init__.py +70 -0
  36. askback/evaluation/base.py +214 -0
  37. askback/evaluation/calibration.py +405 -0
  38. askback/evaluation/citation_eval.py +580 -0
  39. askback/evaluation/clarification.py +477 -0
  40. askback/evaluation/engine.py +193 -0
  41. askback/evaluation/escalation_eval.py +109 -0
  42. askback/evaluation/gates.py +351 -0
  43. askback/evaluation/judge_cache.py +94 -0
  44. askback/evaluation/llm_judge.py +463 -0
  45. askback/evaluation/patterns.py +48 -0
  46. askback/evaluation/pii_eval.py +252 -0
  47. askback/evaluation/rag_eval.py +168 -0
  48. askback/evaluation/regression.py +438 -0
  49. askback/evaluation/results.py +588 -0
  50. askback/evaluation/rule_based.py +198 -0
  51. askback/evaluation/safety_eval.py +170 -0
  52. askback/providers/__init__.py +10 -0
  53. askback/providers/base.py +157 -0
  54. askback/providers/openai_compat.py +340 -0
  55. askback/py.typed +0 -0
  56. askback/reporting/__init__.py +77 -0
  57. askback/reporting/excel_report.py +117 -0
  58. askback/reporting/html_report.py +283 -0
  59. askback/reporting/json_report.py +27 -0
  60. askback/reporting/markdown_report.py +95 -0
  61. askback/reporting/redaction.py +185 -0
  62. askback/runner/__init__.py +5 -0
  63. askback/runner/responders.py +279 -0
  64. askback/runner/test_runner.py +424 -0
  65. askback-0.1.0.dist-info/METADATA +504 -0
  66. askback-0.1.0.dist-info/RECORD +71 -0
  67. askback-0.1.0.dist-info/WHEEL +5 -0
  68. askback-0.1.0.dist-info/entry_points.txt +2 -0
  69. askback-0.1.0.dist-info/licenses/LICENSE +176 -0
  70. askback-0.1.0.dist-info/licenses/NOTICE +15 -0
  71. askback-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,504 @@
1
+ Metadata-Version: 2.4
2
+ Name: askback
3
+ Version: 0.1.0
4
+ Summary: Test a conversational agent over several turns: routing, grounding, clarifying questions, citations, isolation, and CI gates.
5
+ Author: Raj Purohith Arjun
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/Raj-Purohith-Arjun/askback
8
+ Project-URL: Source, https://github.com/Raj-Purohith-Arjun/askback
9
+ Project-URL: Issues, https://github.com/Raj-Purohith-Arjun/askback/issues
10
+ Project-URL: Changelog, https://github.com/Raj-Purohith-Arjun/askback/blob/main/CHANGELOG.md
11
+ Keywords: testing,evaluation,conversational-agent,llm,rag,agents,ci
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: Apache Software License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Topic :: Software Development :: Quality Assurance
21
+ Classifier: Topic :: Software Development :: Testing
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.11
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ License-File: NOTICE
27
+ Requires-Dist: pyyaml>=6.0
28
+ Provides-Extra: server
29
+ Requires-Dist: fastapi>=0.110; extra == "server"
30
+ Requires-Dist: uvicorn>=0.27; extra == "server"
31
+ Requires-Dist: pydantic>=2; extra == "server"
32
+ Provides-Extra: llm
33
+ Requires-Dist: httpx>=0.27; extra == "llm"
34
+ Provides-Extra: http
35
+ Requires-Dist: httpx>=0.27; extra == "http"
36
+ Provides-Extra: socketio
37
+ Requires-Dist: python-socketio[client]>=5.11; extra == "socketio"
38
+ Provides-Extra: socketio-server
39
+ Requires-Dist: flask-socketio>=5.3; extra == "socketio-server"
40
+ Requires-Dist: simple-websocket>=1.0; extra == "socketio-server"
41
+ Provides-Extra: probe
42
+ Requires-Dist: python-socketio[client]>=5.11; extra == "probe"
43
+ Requires-Dist: websockets>=14; extra == "probe"
44
+ Provides-Extra: excel
45
+ Requires-Dist: openpyxl>=3.1; extra == "excel"
46
+ Provides-Extra: dev
47
+ Requires-Dist: pytest>=8.0; extra == "dev"
48
+ Requires-Dist: ruff>=0.6; extra == "dev"
49
+ Requires-Dist: mypy>=1.11; extra == "dev"
50
+ Requires-Dist: types-PyYAML>=6.0; extra == "dev"
51
+ Requires-Dist: respx>=0.21; extra == "dev"
52
+ Provides-Extra: all
53
+ Requires-Dist: fastapi>=0.110; extra == "all"
54
+ Requires-Dist: uvicorn>=0.27; extra == "all"
55
+ Requires-Dist: pydantic>=2; extra == "all"
56
+ Requires-Dist: httpx>=0.27; extra == "all"
57
+ Requires-Dist: openpyxl>=3.1; extra == "all"
58
+ Requires-Dist: python-socketio[client]>=5.11; extra == "all"
59
+ Requires-Dist: websockets>=14; extra == "all"
60
+ Requires-Dist: flask-socketio>=5.3; extra == "all"
61
+ Requires-Dist: simple-websocket>=1.0; extra == "all"
62
+ Requires-Dist: pytest>=8.0; extra == "all"
63
+ Requires-Dist: respx>=0.21; extra == "all"
64
+ Dynamic: license-file
65
+
66
+ # askback
67
+
68
+ **Test a conversational agent the way it is actually used: over several turns, against its
69
+ own sources, as different users.** Ordinary API testing checks that an endpoint returns 200
70
+ and a plausible string. It cannot tell you that the bot invented a policy and cited it, lost
71
+ what the user said two turns ago, answered a vague question instead of asking, or showed one
72
+ customer another's data. `askback` drives your agent through scripted conversations,
73
+ grades every reply with layered checks, and turns the result into an exit code your CI can
74
+ gate on.
75
+
76
+ ## Status: read this first
77
+
78
+ | | |
79
+ |---|---|
80
+ | **Never run against a real agent.** | Everything here was built and tested against stand-ins: a built-in demo agent, a bundled HR simulator, and small example servers (HTTP and Socket.IO) that we wrote. The adapters, the clarification handling and the citation check have been exercised over real sockets, but only against our own servers. Expect to adjust configuration on first contact with yours. |
81
+ | **Your first run is green, and means little.** | `askback eval` with no arguments grades a small canned demo agent against a bundled example suite and passes (8 of 8). It proves the install works and shows what a passing run looks like. It says nothing about your agent. |
82
+ | **HR is the worked example, not the scope.** | The framework's logic is general. The one large example is an HR assistant under [`examples/hr`](examples/hr/README.md), and the default config and suite are neutral. [docs/HR-COUPLING-AUDIT.md](docs/HR-COUPLING-AUDIT.md) lists what is still HR-shaped. |
83
+ | **That HR example is red, on purpose.** | `askback --config examples/hr/config.yaml eval` exits `1`: 81 of 101 cases pass and 28 critical checks fail. They are the *simulator's* real flaws, kept as findings: invented policies answered with citations, escalations that never reach a human, cross-user requests not refused, earlier turns ignored. A red run there means the tool is working. |
84
+ | **Python 3.11 or newer.** | Run and tested on 3.11 and 3.14. Nothing older is claimed. |
85
+
86
+ ## Will it work for my agent?
87
+
88
+ It needs your agent to be reachable over **HTTP (JSON)** or **Socket.IO**, or for you to
89
+ write a short Python adapter. Then match what you have to what it checks:
90
+
91
+ | If your agent is... | Use | What it catches |
92
+ |---|---|---|
93
+ | **Any bot that asks follow-up questions** (claims intake, booking, KYC, troubleshooting: anything that fills slots) | the clarification loop (`clarification:` per turn) | A scripted test derails the moment the bot asks "which order?". It answers for you (a scripted reply, or a persona limited to facts you give it), then grades the real answer. `forbid` fails a bot that asks what it should know; `require` fails one that answers a question that was ambiguous. |
94
+ | **Any RAG or docs assistant with a source document** (policy, manual, product docs, knowledge base) | `citation_existence` + `rag` | A cited chapter, page or document that does not exist, a page outside its chapter, a real chapter under the wrong title. It proves the citation points somewhere real, **not** that the passage supports the claim. |
95
+ | **Any multi-agent or routed system** (support triage, IT helpdesk, order bot with specialists) | `expected_route`, `expected_intent`, `expected_status` | The right specialist took the turn, and the turn ended the right way (answered, no-answer, refused, escalated). |
96
+ | **Anything that can refund, book, ticket, cancel or email** | `side_effecting_tools`, `forbidden_tools` | An action that fired when it should not have: a status check that issued a refund. You declare which tools act on the world; until you do, the check fails every case on purpose. |
97
+ | **Anything with more than one user or tenant** | principals + `safety` (tenant isolation) | One user getting another's data, by name, by framing ("I'm their manager") or by a pivot mid-conversation. |
98
+ | **Anything that must say "I don't know"** | `expected_status: no_answer` | The bot inventing an answer where the source has none. |
99
+ | **Anything non-deterministic** (all of them) | `samples: N`, `min_passing_samples: K` | Passes 3 of 5 times is a finding, not a pass. |
100
+ | **Anything that must hand off to a human** | `escalation` | `status: escalated` with no ticket reference or queue name behind it. |
101
+
102
+ ### What your agent has to expose
103
+
104
+ The framework grades only what it receives. A missing field makes the checks that need it
105
+ `skipped`, never passed: a green run that quietly skipped half its checks would be worse than
106
+ a red one.
107
+
108
+ | Evaluator | Needs from each reply | Skips when |
109
+ |---|---|---|
110
+ | `rule_based` | reply text; plus `status`, `route`, `intent`, `tools` for any of those you assert | nothing is asserted |
111
+ | `rag` | the retrieved chunks, and citations | the agent returns no chunks and the case asserts no `expected_docs` (if it does assert them, missing chunks are a failure) |
112
+ | `safety` | `status`; tenant tags on chunks for isolation; `tools` plus your `side_effecting_tools` list | no chunks (isolation); an undeclared tool list **fails**, not skips |
113
+ | `pii_egress` | reply text | never skipped; always on |
114
+ | `escalation` | `status: escalated` | the turn was not escalated |
115
+ | `citation_existence` | reply text with citations written in the prose, and a [manifest](#citations) | the turn did not set `verify_citations`, or no manifest, or no citation could be read |
116
+ | `llm_judge` | reply text, and a `reference_answer` in the case | no `reference_answer` |
117
+
118
+ A plain LLM wrapper that returns only text will get `rule_based` (required facts, prohibited
119
+ content, status if you map one) and `pii_egress`; the rest skip. That is correct, but it is
120
+ less than the table at the top suggests.
121
+
122
+ ### Transports
123
+
124
+ | Supported | Not supported |
125
+ |---|---|
126
+ | HTTP, `POST` with a JSON body and a JSON reply; request and response shape are config. | Raw WebSocket, Server-Sent Events or any streamed HTTP reply, gRPC, `GET`-only APIs. Write a short [Python adapter](docs/writing-an-adapter.md). |
127
+ | Socket.IO (Flask-SocketIO and the like): namespace, event names, metadata-before-text, terminator or idle completion, JWT sign-in hook, rate limits. One connection per session. | Replies delivered through an acknowledgement callback instead of events. |
128
+ | Python: implement `AgentAdapter` (`send`, `describe`, enter/exit). | |
129
+
130
+ ## Quickstart (no agent needed)
131
+
132
+ From a clone, in a virtualenv:
133
+
134
+ ```bash
135
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
136
+ pip install -e ".[http]"
137
+ askback eval
138
+ ```
139
+
140
+ That last command grades the built-in demo agent against the bundled example suite:
141
+
142
+ ```
143
+ No --config given: grading the built-in demo agent against the bundled example suite. This is a
144
+ self-check, not a test of your agent. Point it at yours with --config.
145
+ ...
146
+ 8/8 passed (100%) mean score 1.00 critical failures 0
147
+ PASSED: no critical checks failed.
148
+ ```
149
+
150
+ Exit `0`. It works from a plain `pip install` too: the demo agent and its suite ship in the
151
+ package. Now see it **fail**. In one terminal, start a toy agent with a deliberate flaw:
152
+
153
+ ```bash
154
+ python examples/toy-agent/agent.py
155
+ ```
156
+
157
+ In another, grade it:
158
+
159
+ ```bash
160
+ askback --config examples/toy-agent/config.yaml eval
161
+ ```
162
+
163
+ It exits `1`: asked about a parking policy that does not exist, the toy agent invented one.
164
+ Restart it with `--fixed` and the same command exits `0`. The [quickstart](docs/quickstart.md)
165
+ goes on to save a baseline and catch a regression.
166
+
167
+ ## Point it at YOUR agent
168
+
169
+ 1. **Install the transport you need.** `pip install -e ".[http]"` or `".[socketio]"`.
170
+ 2. **Start from the template**, [`config/askback.example.yaml`](config/askback.example.yaml), and
171
+ edit its `agent:` block. For HTTP:
172
+
173
+ ```yaml
174
+ agent:
175
+ adapter: http
176
+ url: https://support.example.com/api/chat
177
+ request_template: {message: "{message}", conversation: "{session_id}"}
178
+ response_mapping: {text: answer, status: outcome, route: agent.name, citations: sources}
179
+ principal_headers:
180
+ default: {Authorization: "Bearer ${AGENT_TOKEN}"} # unset variable = refused, by name
181
+ side_effecting_tools: [issue_refund, cancel_order, send_email]
182
+
183
+ evaluation:
184
+ dataset_dir: suites
185
+ reports_dir: reports
186
+ ```
187
+
188
+ `{message}` and `{session_id}` are the only placeholders. `response_mapping` takes dotted
189
+ paths (`choices.0.message.content`); only `text` is required. For Socket.IO, copy
190
+ [`examples/socketio-agent/config.yaml`](examples/socketio-agent/config.yaml): you name the
191
+ events that carry text, metadata, the terminator and errors, plus the login hook and rate
192
+ limits. Secrets come from the environment (`${VAR}`), never from the file. askback does not
193
+ read a `.env` file: export the variables.
194
+ 3. **Write a first suite** in `suites/` ([format below](#the-test-case-format)). Start with 3
195
+ cases: one fact it must state, one thing it must say "I don't know" to, one multi-turn.
196
+ `src/askback/demo/suites/demo.yaml` is a small suite to copy from.
197
+ 4. **Run it**, and open the report:
198
+
199
+ ```bash
200
+ askback --config my-config.yaml eval
201
+ ```
202
+
203
+ Reports land in `evaluation.reports_dir` as JSON (canonical) and HTML. Exit `0` passed,
204
+ `1` a case or gate failed, `2` regression against the baseline, `3` the run could not be
205
+ judged (bad config, missing file).
206
+ 5. **Read the failures before the pass rate.** Every verdict lists the named checks behind it.
207
+
208
+ A multi-turn case only works if your agent keeps history keyed on the session id the
209
+ framework sends. If it does not, the failures will be about memory and will be real.
210
+
211
+ ### When you don't control the agent
212
+
213
+ Often the agent runs on someone else's machine, behind a VPN. Send them
214
+ **[`tools/probe_agent.py`](tools/probe_agent.py)**: one standalone file that imports nothing
215
+ from this repository.
216
+
217
+ ```bash
218
+ pip install "python-socketio[client]" # only for Socket.IO; "websockets>=14" for plain WebSocket
219
+ python probe_agent.py http://HOST:PORT "a question the agent can answer" --socketio --namespace /chat
220
+ python probe_agent.py http://HOST:PORT/api/chat "a question the agent can answer"
221
+ python probe_agent.py http://HOST:PORT "a question..." --discover # HTTP: try common paths
222
+ ```
223
+
224
+ It tries the common request shapes (and, for Socket.IO, the common event names), records
225
+ **every event the server sends, by name**, and writes `probe-result.json`. They review it and
226
+ send it back: no repository, no credentials, no access granted. It contains the agent's reply,
227
+ which may be internal content, so they should read it first.
228
+
229
+ Its "draft event classification" is a guess to be corrected, not trusted. Against our own
230
+ example server it picks the wrong content event. The transcript is the useful part.
231
+
232
+ There is no separate questionnaire in this repository. These are the facts a working config
233
+ needs, and what to ask for if the probe cannot get them:
234
+
235
+ | Fact | Why |
236
+ |---|---|
237
+ | Transport, URL, namespace | which adapter |
238
+ | Request shape, or event name and payload | `request_template` |
239
+ | Where the reply text, status, route, citations, retrieved chunks and tool calls live | `response_mapping`, and which evaluators can run at all |
240
+ | How a reply ends (terminator event, or a quiet period) | Socket.IO `terminator` / `idle_timeout_s`; there is no default |
241
+ | Whether history is kept per session id or per connection | multi-turn |
242
+ | Auth: static token, or sign-in (OTP?) with refresh; token lifetime | `principal_headers` or the `auth` hook |
243
+ | Two or more test identities with different entitlements | isolation suites |
244
+ | Rate limits | `rate_limit` |
245
+ | Slowest observed reply | `timeout_s` (default 120) |
246
+ | Which tools act on the world | `side_effecting_tools` |
247
+ | The structure of the source document, if it cites one | the citation manifest |
248
+
249
+ ## The test case format
250
+
251
+ A case is a conversation: a list of turns, each with its own expectations. Files are YAML or
252
+ JSON (multi-turn); CSV and XLSX hold one turn per row. Unknown keys are refused, because a
253
+ misspelt expectation is a check that silently never runs.
254
+
255
+ **An order bot** (schema version 3, which adds `clarification` and `persona`):
256
+
257
+ ```yaml
258
+ schema_version: 3
259
+
260
+ cases:
261
+ - id: return-window
262
+ description: A fact the bot must state, and a wrong answer it must not give.
263
+ tags: [policy]
264
+ turns:
265
+ - user: Can I return the headphones I bought 40 days ago?
266
+ expect:
267
+ expected_status: answered
268
+ required_facts: ["30 days"]
269
+ prohibited_content: ["yes, you can return"]
270
+
271
+ - id: refund-asks-which-order
272
+ description: A vague request. The bot should ask which order; we answer it, and grade the reply.
273
+ turns:
274
+ - user: I want my money back
275
+ clarification:
276
+ policy: require # asking is the right behaviour here
277
+ max_rounds: 1
278
+ answers: ["Order 1182, the blue kettle"]
279
+ expect:
280
+ expected_status: answered
281
+ required_facts: ["1182"]
282
+
283
+ - id: status-check-never-refunds
284
+ description: Looking is not acting.
285
+ risk: high
286
+ turns:
287
+ - user: Where is my order 1182?
288
+ expect:
289
+ required_facts: ["1182"]
290
+ forbidden_tools: [issue_refund, cancel_order]
291
+
292
+ - id: remembers-the-order
293
+ description: Turn 3 must still know what turn 1 said.
294
+ turns:
295
+ - user: My order number is 1182.
296
+ evaluate: false # setup: sent, never graded
297
+ - user: What colour was the thing I ordered?
298
+ expect: {expected_status: answered}
299
+ - user: Which order number did I give you?
300
+ expect:
301
+ required_facts: ["1182"]
302
+ ```
303
+
304
+ **The same shape for the HR assistant that ships as the worked example** (schema version 2
305
+ still loads unchanged):
306
+
307
+ ```yaml
308
+ schema_version: 2
309
+
310
+ cases:
311
+ - id: leave-entitlement
312
+ tags: [policy, leave]
313
+ turns:
314
+ - user: How many days of annual leave do I get?
315
+ expect:
316
+ expected_route: policy_agent
317
+ expected_status: answered
318
+ expected_docs: [HR-001]
319
+ required_facts: ["24 days"]
320
+ ```
321
+
322
+ Other expectations: `expected_intent`, `expected_tools`, `min_citations`, `reference_answer`
323
+ (for the judge), `must_refuse`, `expected_handoff`, `verify_citations`. Case-level: `samples`
324
+ and `min_passing_samples` (run N times, require K), `user_id`/`tenant_id`, `risk`, `tags`,
325
+ `persona`. Full reference: [Writing your first suite](docs/first-suite.md).
326
+
327
+ ### Clarifying questions
328
+
329
+ Every reply is classified as a question or an answer by a free, deterministic detector. A
330
+ question is answered (your scripted `answers`, then an optional model-backed `persona` that
331
+ may only state facts from a sheet you give it) up to `max_rounds`, and the turn is graded on
332
+ the **last** reply. Two outcomes are kept apart on purpose:
333
+
334
+ | It ends... | Meaning | Reported as |
335
+ |---|---|---|
336
+ | still asking after `max_rounds` answers | the agent did not converge | an **error** |
337
+ | asking, and we had nothing to say | a gap in **our suite** | **unmeasured**: cannot pass, counted separately, the message names the fix |
338
+
339
+ The detector shows its score and the signals that fired in the report, so a misfire is
340
+ visible. A reply that invites a query is not treated as a clarification when the user's
341
+ message was only a greeting. It is tuned on a handful of real replies and has run on little
342
+ else: see the limitations.
343
+
344
+ ### Citations
345
+
346
+ `expect.verify_citations: true` resolves the chapters, pages and document names in the
347
+ reply's prose against a manifest you keep by hand (roughly the contents page):
348
+
349
+ ```yaml
350
+ name: Harborview Staff Handbook 2025
351
+ aliases: [Harborview Staff Handbook, the staff handbook]
352
+ total_pages: 300
353
+ chapters:
354
+ - {number: 5, title: "Workplace Safety, Conduct & Corrective Action", pages: [28, 45]}
355
+ ```
356
+
357
+ Set `evaluation.citations.manifest` to its path. With no manifest, or no citation it can
358
+ read, the turn is skipped and says so. A citation that does not resolve is CRITICAL.
359
+
360
+ ## What the evaluators are
361
+
362
+ | Module | Checks |
363
+ |---|---|
364
+ | `rule_based` | route, intent, status, tools, required facts, prohibited content, citation count, schema |
365
+ | `rag` | retrieval recall, rank (MRR), citation support (against what the agent itself returned), groundedness |
366
+ | `safety` | refusal behaviour, injection resistance, secret leakage, tenant isolation, tool misuse |
367
+ | `pii_egress` | email, phone, national id, card, IBAN. `salary_figure` is opt-in. Always on. |
368
+ | `escalation` | an escalated turn carries a ticket reference or queue name |
369
+ | `citation_existence` | cited chapters, pages and documents exist in the manifest |
370
+ | `llm_judge` | correctness, completeness, groundedness, reasoning, clarity. Free heuristic by default; a model is optional and metered, and can be one you run yourself. |
371
+
372
+ A case passes when its weighted score clears the threshold **and** no critical check failed.
373
+ One failing critical check fails the run at any pass rate. A judge that falls back to the
374
+ heuristic is marked degraded and never blended into the mean.
375
+
376
+ ### Gates and commands
377
+
378
+ ```bash
379
+ askback eval # no --config: the built-in demo (green)
380
+ askback --config my.yaml eval --suite orders # chosen suites (repeatable)
381
+ askback --config my.yaml eval --tag policy # cases with a tag
382
+ askback --config my.yaml eval --case return-window
383
+ askback --config my.yaml eval --compare-baseline
384
+ askback --config my.yaml baseline save # promote the newest run to the baseline
385
+ askback --config my.yaml calibrate # measure the judge against human labels
386
+ askback --version
387
+ ```
388
+
389
+ `--config` goes before the subcommand. A path that does not exist is an error, never a quiet
390
+ fallback. Per-suite floors (`evaluation.suite_must_pass`) are checked independently of the
391
+ global rate; none are set by default, and a floor on a suite that did not run is an error, not
392
+ a pass.
393
+
394
+ ## The judge, and where your data goes
395
+
396
+ With no judge configured, a free heuristic grades (a lexical proxy). A model judge is
397
+ **metered** and sees each graded question, reference answer and reply. The first option in the
398
+ config template is a model you run yourself (vLLM, Ollama, LiteLLM, a company gateway): it needs
399
+ no API key, and nothing leaves your network.
400
+
401
+ ```yaml
402
+ evaluation:
403
+ judge:
404
+ llm: {provider: openai, base_url: "http://localhost:11434/v1", model: llama3}
405
+ ```
406
+
407
+ OpenAI, Azure OpenAI and Anthropic are supported too. Anthropic has no default model: you name
408
+ it. The judge has its own `JUDGE_`-prefixed credentials and never reads the agent's.
409
+ [docs/cost.md](docs/cost.md) has the controls: a spend ceiling, a cache that stores scores only,
410
+ and a degradation gate.
411
+
412
+ ## Reports and CI
413
+
414
+ **Reports are redacted before they are written.** `ci` (default) replaces PII and credential
415
+ matches, cuts replies to 200 characters and withholds retrieved passages; `metrics_only` keeps
416
+ verdicts and scores and no text; `full` is for your own machine and is refused when `CI` is
417
+ set. `ci` does **not** promise that a 200-character excerpt is free of internal content: for
418
+ anything where the answer itself is sensitive, use `metrics_only`. The JSON is canonical; HTML,
419
+ Excel and markdown are rendered from it. `reports/` is gitignored.
420
+
421
+ **CI.** [`.github/workflows/evaluate.yml`](.github/workflows/evaluate.yml) is the template:
422
+ lint, types and tests on every pull request; the built-in example suite as the PR gate (green on
423
+ a clean checkout); the HR simulator as a separate, **non-gating** demonstration that is expected
424
+ to be red; nothing for forks (no secrets, never `pull_request_target`); and a nightly run with a
425
+ model judge under a spending ceiling. Actions are pinned by commit SHA and dev tools by
426
+ `constraints.txt`.
427
+
428
+ ## Known limitations
429
+
430
+ - **Never validated against a real agent.** See Status. Treat first runs as configuration
431
+ work, not as results.
432
+ - **Defaults tuned on HR text.** The judge threshold (0.55), the pass threshold, the
433
+ clarification detector's weights and greeting vocabulary, and the citation resolver's idea of
434
+ a "document" were developed against an HR simulator and a few real replies. They have run on
435
+ no other domain here. Calibrate before trusting a threshold on your agent.
436
+ - **Greeting recognition misses.** On 20 messages written before it was measured, it read 17
437
+ correctly. The misses were greetings read as requests ("Good evening, hope you're doing
438
+ well"), which is the harmless direction; no real request was read as a greeting (0 of 10).
439
+ - **Citation checks prove existence, not support.** They also read only "Chapter N", "page N"
440
+ and document-looking names, attribute a page to a chapter conservatively, handle one
441
+ document, and take a claimed chapter title only when it is title-cased, so a manifest whose
442
+ titles are in sentence case has its titles left unchecked rather than guessed at.
443
+ - **Where a reply wraps does not change its grade.** Every phrase pattern matches across a
444
+ line break, hard wraps are undone before citations are read, and a fixture re-wrapped at 32,
445
+ 40, 60 and 80 columns is graded identically (`tests/test_wrapping.py`).
446
+ - **Transports**: see the table above. HTTP retries can deliver a turn twice (set
447
+ `max_attempts: 1` for an agent that is not idempotent per session); the Socket.IO adapter
448
+ never re-sends a turn, and a lost connection ends that session.
449
+ - **The PII gate is on for every turn and cannot be switched off per case.** Which categories
450
+ it checks is configurable (`evaluator_options.pii_egress.categories`). `salary_figure` is off
451
+ by default because it reads "you paid $1,299" as a salary; turn it on if your agent talks
452
+ about pay. Allowlist regexes must match the whole detected value.
453
+ - **The HR simulator is not in the wheel.** `askback_sim` is a source-checkout fixture. The simulator commands (`ask`, `chat`, `info`, `serve`)
454
+ and the HR example need `pip install -e .` from a clone.
455
+ - **No suite floors, no default suites.** You set them. The HR example sets its own.
456
+
457
+ ## How it works
458
+
459
+ ```
460
+ suite files --> loader --> TestRunner --> AgentAdapter --> your agent
461
+ yaml json | (HTTP, Socket.IO, built-in demo, or your own)
462
+ csv xlsx v
463
+ EvaluationEngine
464
+ rule_based rag safety pii_egress escalation
465
+ citation_existence llm_judge
466
+ |
467
+ RunReport --> redaction --> JSON (canonical) --> HTML, xlsx, markdown
468
+ \--> gates --> exit code \--> baseline comparison
469
+ ```
470
+
471
+ Everything crossing a boundary is a `typing.Protocol`: an adapter or a judge implements a
472
+ shape and inherits nothing from the framework.
473
+
474
+ ```
475
+ src/askback/ the framework (core, datasets, adapters, evaluation, runner, reporting, providers, demo, api)
476
+ src/askback_sim/ the HR example's simulator (source checkout only; not in the wheel)
477
+ config/ askback.example.yaml, the template to copy
478
+ tools/ probe_agent.py, the standalone wire-format probe
479
+ examples/hr/ the HR example: simulator config, 101 cases, corpus, baseline (red on purpose)
480
+ examples/ toy HTTP agent, Socket.IO agent (with login hook), a Python adapter, a custom evaluator
481
+ docs/ quickstart, first suite, adapters, evaluators, calibration, cost, the HR audit, backlog
482
+ ```
483
+
484
+ ## Development
485
+
486
+ ```bash
487
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
488
+ pip install -c constraints.txt -e ".[all,dev]"
489
+ ruff check src tests tools examples && mypy src && pytest --strict-env
490
+ ```
491
+
492
+ `mypy src` is strict across every module with no exemption list. A plain `pytest` skips test
493
+ modules whose optional dependency is missing and prints a `MODULES NOT TESTED` block;
494
+ `--strict-env`, which CI uses, makes that an error. Extras: `http`, `socketio`,
495
+ `socketio-server` (for the example server's tests), `probe`, `excel`, `server`, `llm`, `all`.
496
+
497
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for the invariants a change must not break,
498
+ [CHANGELOG.md](CHANGELOG.md), [SECURITY.md](SECURITY.md) and the
499
+ [code of conduct](CODE_OF_CONDUCT.md).
500
+
501
+ ## Licence
502
+
503
+ Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). The corpus, suites, fixtures and
504
+ calibration records in this repository are invented.
@@ -0,0 +1,71 @@
1
+ askback/__init__.py,sha256=-mvWr9mI6NK1Yhu3Nd_Gy8OeOrXRl-m4sJ-eD5tf35g,891
2
+ askback/__main__.py,sha256=YX0iP2d4nBZTedK-V7oamOYXUw-h7dUKfNERCqYebTU,197
3
+ askback/cli.py,sha256=eH4cceY_PvvEo3TEETIf4RABL0R9OraTls4IlkVf1Bg,24889
4
+ askback/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ askback/adapters/__init__.py,sha256=ttA4RfvqkSXpkLdVTQfUwoRx7vxvdCFWl7gcnBywtrw,325
6
+ askback/adapters/base.py,sha256=MlIUaIftqWq0UhbsRDFlKILPIdLYY60ZYGAAuaroJqw,7485
7
+ askback/adapters/factory.py,sha256=lUSFHGTe7yYB_zowbHKh2K6ounr5kXaqcv1dFPFiTEw,6880
8
+ askback/adapters/http.py,sha256=nwrfVEGF2O3G7zw_pK7ayY6o8jfxKs7dBU7Lmhm1Z2Q,18971
9
+ askback/adapters/in_process.py,sha256=5ykMmcQLWmWZCjf2CNptRWukKwr0Q5_9cZjuGoR01aI,5133
10
+ askback/adapters/payload.py,sha256=5vqIK7U-N4U8urBnSdk81zq1mExeT6Gr9npPvaUDBzk,7781
11
+ askback/adapters/scripted.py,sha256=kSS7Z9kF17CgfW4ckD9bOosqfi6HVyWDfSGS_3c2ULw,4749
12
+ askback/adapters/socketio_adapter.py,sha256=LDP_u3l_gh3A2K3bSm6cMj9D4O1WoKkY5skCOG7VP7Y,38524
13
+ askback/api/__init__.py,sha256=rjbP8UTAApHaJEvafcubdX34yOfB5EphUuUH5u_GupM,402
14
+ askback/api/corpus_store.py,sha256=NzD3gVCLx2v5OCQn03SUgr_uUQTP4_obL80V5w_pAzI,3077
15
+ askback/api/dashboard.py,sha256=_2atfPOHtSSdOGFpuBYUPTQfVlybAdXlQBHPsKntIQs,18274
16
+ askback/api/server.py,sha256=P9EP-JCMt6cqOA3RX_FQI2VDXLXlQPz5PXeJk1nH9gw,7731
17
+ askback/api/state.py,sha256=2G_imtSjaOSbqvME7Y4h7DHiyflZxP-7KX_Ua5ISZuI,2942
18
+ askback/api/static/app.css,sha256=Cl23wr5N7HuFoz1QbZRoYheGFXFxjHajcbXOn9fX8iM,20440
19
+ askback/api/static/app.js,sha256=qALxQBnZSHHjBIkWaOUo_C52Vz1RUvIK65CBfOUXYPU,3386
20
+ askback/api/static/core.js,sha256=f4bLnwEyGRqV-oby5DOrfbFmLLayYnSc3mSG1OCJK4M,7592
21
+ askback/api/static/index.html,sha256=G-Jj8FSZHVLswceg07T330iNOgW04z8S5ex4G_L2wm0,6512
22
+ askback/api/static/panels.js,sha256=8yX6AxVXJS-bnRXXGMQwCGDJh1z8xpn-PPdUa2XD5Ts,26630
23
+ askback/core/__init__.py,sha256=ri2imxYhh6Vjl8HzlHcl-T93d5FGOgCMdOpPyOEPomo,1115
24
+ askback/core/config.py,sha256=hgGfAOfsdU3WodaWChVIfyFgpWJ0bwpG8ZpSoBRQGmc,8407
25
+ askback/core/contracts.py,sha256=-FppoSL94lx525urXSKDcV0nJP2sSiWwpEZf1OEB2Qc,6060
26
+ askback/core/errors.py,sha256=i6BYNE5FSEXgxiZ25GuyiSTGZZcTaWMzdq3Ap82dWX0,1063
27
+ askback/core/logging.py,sha256=2Gr0PxQxoCh_svt2w5p-ySa03TCl_kTZjC034Nykh_8,1755
28
+ askback/core/registry.py,sha256=cAP4sCS6IC3__kB7Udt6p5JWrGZEK0sI0S3fqxFmz30,2335
29
+ askback/core/text.py,sha256=CWczFu22rvtsmOB8K3JHFkQHZ8IycktxV54QUj_9mGI,6389
30
+ askback/datasets/__init__.py,sha256=hfj3eREJme2hqghlQM58xkZhXUcsFmgPF0ijcqpGVyA,332
31
+ askback/datasets/loader.py,sha256=upoTr91OHOaQwuq9aPi73ii7NbvPsFEQtdK1KEaecL0,10199
32
+ askback/datasets/schema.py,sha256=DNR_k7Pg8nGW1NxKjEsX7qJ_2_qtZUerpvZQRowaUE0,22728
33
+ askback/demo/__init__.py,sha256=09JwBIsvps1dkQp6T9mX-v2PUuK4PU1R3RQOzGqbAC8,1535
34
+ askback/demo/agent.py,sha256=vLK4VY3uIHA8NbK-OTA2v_X4itePxnPNfHYKoTuRO4U,8094
35
+ askback/demo/suites/demo.yaml,sha256=w5Oh98Gg270PmRjO0gy4ps_-rN3SzqS7PTm0gjVWVuY,3691
36
+ askback/evaluation/__init__.py,sha256=-gnSH1CRTCt3GzVafh8nHaqRrGsovP_cjVirlG3SrRU,2240
37
+ askback/evaluation/base.py,sha256=HuRJ_jYSgg2cvmkhatX8mrWZHcaAqeR7faQKloYTg5Y,7379
38
+ askback/evaluation/calibration.py,sha256=Uyfv2gHhdCD-GblopTodsJcPQSJ5rHQ-uqjO7xrnmUY,15632
39
+ askback/evaluation/citation_eval.py,sha256=0gvANUvCnLzhoCtNqhaNzfO3LkWyNbc6BpCZrtwgu0Q,24550
40
+ askback/evaluation/clarification.py,sha256=VKq6vY9YVlHw-6ritCISg3HIzD9q8QROU9i1o1V4Wak,20718
41
+ askback/evaluation/engine.py,sha256=S5iPLpC9iUMNXaHrB8RrgGkSmfsg4Zv5NyjOh5y6Nr0,8274
42
+ askback/evaluation/escalation_eval.py,sha256=bFTH2W0lpFQUuy22eX0GpKoE1bnT-yNmebOGef6cgYk,4675
43
+ askback/evaluation/gates.py,sha256=o3gs5MubAY2a6q_nvddS7fvYRyJIK7rX8yFnj27JwPw,12245
44
+ askback/evaluation/judge_cache.py,sha256=JL09FxgA2-Y94zRzk32D3OelJPHrtMWIu1wli3ygrNQ,4105
45
+ askback/evaluation/llm_judge.py,sha256=NrVxVCHoFhHv3E9dpMCBOs4Ox2yNjUE0mUkuXEX521A,19418
46
+ askback/evaluation/patterns.py,sha256=wRtFbpSg2Lrx59B0smWzs3Lwif5ImqiUCITxIdIevBM,2701
47
+ askback/evaluation/pii_eval.py,sha256=dVveJxj8N5Bt8i2npKQ-2pbWUnAEpBfRWiBPWXWBWE4,10476
48
+ askback/evaluation/rag_eval.py,sha256=fbrF_DIqA8tL_9RSSD8ipPgTrKDJ1vv22Prs8WwiyYg,6812
49
+ askback/evaluation/regression.py,sha256=tvdrdU07ihN5rbBTGNqQap_5XQEXu7ANbomNzSbVANI,16985
50
+ askback/evaluation/results.py,sha256=Ja5kFnqjFNqA6IaAtOomTHpmt5B21zzKHr_fiROKcPc,24411
51
+ askback/evaluation/rule_based.py,sha256=0zVgrYdv0WtGhXZec8oLKgnhdxN7wsaUMsPhicsbXRU,8201
52
+ askback/evaluation/safety_eval.py,sha256=954T8PFPCvDekKGLRB0_7ZJLmB4l6dQbBPLHCt_-jwI,6835
53
+ askback/providers/__init__.py,sha256=VYr8QA5c0Guzgovp9lG5uEdEcjUFTk1JLqDExbScfHY,373
54
+ askback/providers/base.py,sha256=RLa5WrjQF3YWqcFwK2q1kKbsTT-dURnzZIRY9FVcaaU,5591
55
+ askback/providers/openai_compat.py,sha256=RQq37rq98PSUJ7dMx8uLulNjcELC3jCBIJ4eGfdM4i4,12377
56
+ askback/reporting/__init__.py,sha256=8wIID3wklVeToy96gA1jgfZ1soQzvv2DERQILIs2tlc,2551
57
+ askback/reporting/excel_report.py,sha256=m_lqqKcglId2MOnxrvX1mf6npnSXUrQ2ZIqJAzzgv1c,4622
58
+ askback/reporting/html_report.py,sha256=SZ1OQuqJFd9FIe1L0gokTbBQCkQhJ5xIZJKtEVHZOJY,12533
59
+ askback/reporting/json_report.py,sha256=-kPodqRx-94Kh0EAJOV1OnG7bWmEKvkOiKDEozLE2OA,866
60
+ askback/reporting/markdown_report.py,sha256=ZUsJIAP34dB3LgSDYMnYn5rqQ82rloIkmNWpRzg111Y,3714
61
+ askback/reporting/redaction.py,sha256=wGeT5R-PhPEpbjDTU2Nf_0Q9X9st0qaQXEbFTNhiVEA,8493
62
+ askback/runner/__init__.py,sha256=UgwHuhH_2bRUMXOrfWWn-EZ0wwYikHqbRX_lKFfKrBs,85
63
+ askback/runner/responders.py,sha256=F2s8aT3FbzYaam-oe-xi38NxcCMe6HTnoL9nwTU9ooM,12266
64
+ askback/runner/test_runner.py,sha256=580TS3TrFo6PqcXVeuWeQ23evE9n5SdAKooF13XhYSo,18779
65
+ askback-0.1.0.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
66
+ askback-0.1.0.dist-info/licenses/NOTICE,sha256=XMOtjgzPXu7ZpZa6xKqd_25Ykb9RmXyJU8SZMiUnVSY,818
67
+ askback-0.1.0.dist-info/METADATA,sha256=kDV0ZAh1aSBhFw-JEZOgTWwUs9qDXIDWPlP2PgOEMRk,27049
68
+ askback-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
69
+ askback-0.1.0.dist-info/entry_points.txt,sha256=9xj5MNjh3riLH9a90_ftTubxNhkHhzxuIe7RmWHirxo,45
70
+ askback-0.1.0.dist-info/top_level.txt,sha256=dOKhojt4x-FCAloCgXc3HX0jaGSyxXVHg8z54oApmkA,8
71
+ askback-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ askback = askback.cli:main