askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: askback
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Test a conversational agent over several turns: routing, grounding, clarifying questions, citations, isolation, and CI gates.
|
|
5
|
+
Author: Raj Purohith Arjun
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/Raj-Purohith-Arjun/askback
|
|
8
|
+
Project-URL: Source, https://github.com/Raj-Purohith-Arjun/askback
|
|
9
|
+
Project-URL: Issues, https://github.com/Raj-Purohith-Arjun/askback/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/Raj-Purohith-Arjun/askback/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: testing,evaluation,conversational-agent,llm,rag,agents,ci
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
21
|
+
Classifier: Topic :: Software Development :: Testing
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
License-File: NOTICE
|
|
27
|
+
Requires-Dist: pyyaml>=6.0
|
|
28
|
+
Provides-Extra: server
|
|
29
|
+
Requires-Dist: fastapi>=0.110; extra == "server"
|
|
30
|
+
Requires-Dist: uvicorn>=0.27; extra == "server"
|
|
31
|
+
Requires-Dist: pydantic>=2; extra == "server"
|
|
32
|
+
Provides-Extra: llm
|
|
33
|
+
Requires-Dist: httpx>=0.27; extra == "llm"
|
|
34
|
+
Provides-Extra: http
|
|
35
|
+
Requires-Dist: httpx>=0.27; extra == "http"
|
|
36
|
+
Provides-Extra: socketio
|
|
37
|
+
Requires-Dist: python-socketio[client]>=5.11; extra == "socketio"
|
|
38
|
+
Provides-Extra: socketio-server
|
|
39
|
+
Requires-Dist: flask-socketio>=5.3; extra == "socketio-server"
|
|
40
|
+
Requires-Dist: simple-websocket>=1.0; extra == "socketio-server"
|
|
41
|
+
Provides-Extra: probe
|
|
42
|
+
Requires-Dist: python-socketio[client]>=5.11; extra == "probe"
|
|
43
|
+
Requires-Dist: websockets>=14; extra == "probe"
|
|
44
|
+
Provides-Extra: excel
|
|
45
|
+
Requires-Dist: openpyxl>=3.1; extra == "excel"
|
|
46
|
+
Provides-Extra: dev
|
|
47
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
48
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
49
|
+
Requires-Dist: mypy>=1.11; extra == "dev"
|
|
50
|
+
Requires-Dist: types-PyYAML>=6.0; extra == "dev"
|
|
51
|
+
Requires-Dist: respx>=0.21; extra == "dev"
|
|
52
|
+
Provides-Extra: all
|
|
53
|
+
Requires-Dist: fastapi>=0.110; extra == "all"
|
|
54
|
+
Requires-Dist: uvicorn>=0.27; extra == "all"
|
|
55
|
+
Requires-Dist: pydantic>=2; extra == "all"
|
|
56
|
+
Requires-Dist: httpx>=0.27; extra == "all"
|
|
57
|
+
Requires-Dist: openpyxl>=3.1; extra == "all"
|
|
58
|
+
Requires-Dist: python-socketio[client]>=5.11; extra == "all"
|
|
59
|
+
Requires-Dist: websockets>=14; extra == "all"
|
|
60
|
+
Requires-Dist: flask-socketio>=5.3; extra == "all"
|
|
61
|
+
Requires-Dist: simple-websocket>=1.0; extra == "all"
|
|
62
|
+
Requires-Dist: pytest>=8.0; extra == "all"
|
|
63
|
+
Requires-Dist: respx>=0.21; extra == "all"
|
|
64
|
+
Dynamic: license-file
|
|
65
|
+
|
|
66
|
+
# askback
|
|
67
|
+
|
|
68
|
+
**Test a conversational agent the way it is actually used: over several turns, against its
|
|
69
|
+
own sources, as different users.** Ordinary API testing checks that an endpoint returns 200
|
|
70
|
+
and a plausible string. It cannot tell you that the bot invented a policy and cited it, lost
|
|
71
|
+
what the user said two turns ago, answered a vague question instead of asking, or showed one
|
|
72
|
+
customer another's data. `askback` drives your agent through scripted conversations,
|
|
73
|
+
grades every reply with layered checks, and turns the result into an exit code your CI can
|
|
74
|
+
gate on.
|
|
75
|
+
|
|
76
|
+
## Status: read this first
|
|
77
|
+
|
|
78
|
+
| | |
|
|
79
|
+
|---|---|
|
|
80
|
+
| **Never run against a real agent.** | Everything here was built and tested against stand-ins: a built-in demo agent, a bundled HR simulator, and small example servers (HTTP and Socket.IO) that we wrote. The adapters, the clarification handling and the citation check have been exercised over real sockets, but only against our own servers. Expect to adjust configuration on first contact with yours. |
|
|
81
|
+
| **Your first run is green, and means little.** | `askback eval` with no arguments grades a small canned demo agent against a bundled example suite and passes (8 of 8). It proves the install works and shows what a passing run looks like. It says nothing about your agent. |
|
|
82
|
+
| **HR is the worked example, not the scope.** | The framework's logic is general. The one large example is an HR assistant under [`examples/hr`](examples/hr/README.md), and the default config and suite are neutral. [docs/HR-COUPLING-AUDIT.md](docs/HR-COUPLING-AUDIT.md) lists what is still HR-shaped. |
|
|
83
|
+
| **That HR example is red, on purpose.** | `askback --config examples/hr/config.yaml eval` exits `1`: 81 of 101 cases pass and 28 critical checks fail. They are the *simulator's* real flaws, kept as findings: invented policies answered with citations, escalations that never reach a human, cross-user requests not refused, earlier turns ignored. A red run there means the tool is working. |
|
|
84
|
+
| **Python 3.11 or newer.** | Run and tested on 3.11 and 3.14. Nothing older is claimed. |
|
|
85
|
+
|
|
86
|
+
## Will it work for my agent?
|
|
87
|
+
|
|
88
|
+
It needs your agent to be reachable over **HTTP (JSON)** or **Socket.IO**, or for you to
|
|
89
|
+
write a short Python adapter. Then match what you have to what it checks:
|
|
90
|
+
|
|
91
|
+
| If your agent is... | Use | What it catches |
|
|
92
|
+
|---|---|---|
|
|
93
|
+
| **Any bot that asks follow-up questions** (claims intake, booking, KYC, troubleshooting: anything that fills slots) | the clarification loop (`clarification:` per turn) | A scripted test derails the moment the bot asks "which order?". It answers for you (a scripted reply, or a persona limited to facts you give it), then grades the real answer. `forbid` fails a bot that asks what it should know; `require` fails one that answers a question that was ambiguous. |
|
|
94
|
+
| **Any RAG or docs assistant with a source document** (policy, manual, product docs, knowledge base) | `citation_existence` + `rag` | A cited chapter, page or document that does not exist, a page outside its chapter, a real chapter under the wrong title. It proves the citation points somewhere real, **not** that the passage supports the claim. |
|
|
95
|
+
| **Any multi-agent or routed system** (support triage, IT helpdesk, order bot with specialists) | `expected_route`, `expected_intent`, `expected_status` | The right specialist took the turn, and the turn ended the right way (answered, no-answer, refused, escalated). |
|
|
96
|
+
| **Anything that can refund, book, ticket, cancel or email** | `side_effecting_tools`, `forbidden_tools` | An action that fired when it should not have: a status check that issued a refund. You declare which tools act on the world; until you do, the check fails every case on purpose. |
|
|
97
|
+
| **Anything with more than one user or tenant** | principals + `safety` (tenant isolation) | One user getting another's data, by name, by framing ("I'm their manager") or by a pivot mid-conversation. |
|
|
98
|
+
| **Anything that must say "I don't know"** | `expected_status: no_answer` | The bot inventing an answer where the source has none. |
|
|
99
|
+
| **Anything non-deterministic** (all of them) | `samples: N`, `min_passing_samples: K` | Passes 3 of 5 times is a finding, not a pass. |
|
|
100
|
+
| **Anything that must hand off to a human** | `escalation` | `status: escalated` with no ticket reference or queue name behind it. |
|
|
101
|
+
|
|
102
|
+
### What your agent has to expose
|
|
103
|
+
|
|
104
|
+
The framework grades only what it receives. A missing field makes the checks that need it
|
|
105
|
+
`skipped`, never passed: a green run that quietly skipped half its checks would be worse than
|
|
106
|
+
a red one.
|
|
107
|
+
|
|
108
|
+
| Evaluator | Needs from each reply | Skips when |
|
|
109
|
+
|---|---|---|
|
|
110
|
+
| `rule_based` | reply text; plus `status`, `route`, `intent`, `tools` for any of those you assert | nothing is asserted |
|
|
111
|
+
| `rag` | the retrieved chunks, and citations | the agent returns no chunks and the case asserts no `expected_docs` (if it does assert them, missing chunks are a failure) |
|
|
112
|
+
| `safety` | `status`; tenant tags on chunks for isolation; `tools` plus your `side_effecting_tools` list | no chunks (isolation); an undeclared tool list **fails**, not skips |
|
|
113
|
+
| `pii_egress` | reply text | never skipped; always on |
|
|
114
|
+
| `escalation` | `status: escalated` | the turn was not escalated |
|
|
115
|
+
| `citation_existence` | reply text with citations written in the prose, and a [manifest](#citations) | the turn did not set `verify_citations`, or no manifest, or no citation could be read |
|
|
116
|
+
| `llm_judge` | reply text, and a `reference_answer` in the case | no `reference_answer` |
|
|
117
|
+
|
|
118
|
+
A plain LLM wrapper that returns only text will get `rule_based` (required facts, prohibited
|
|
119
|
+
content, status if you map one) and `pii_egress`; the rest skip. That is correct, but it is
|
|
120
|
+
less than the table at the top suggests.
|
|
121
|
+
|
|
122
|
+
### Transports
|
|
123
|
+
|
|
124
|
+
| Supported | Not supported |
|
|
125
|
+
|---|---|
|
|
126
|
+
| HTTP, `POST` with a JSON body and a JSON reply; request and response shape are config. | Raw WebSocket, Server-Sent Events or any streamed HTTP reply, gRPC, `GET`-only APIs. Write a short [Python adapter](docs/writing-an-adapter.md). |
|
|
127
|
+
| Socket.IO (Flask-SocketIO and the like): namespace, event names, metadata-before-text, terminator or idle completion, JWT sign-in hook, rate limits. One connection per session. | Replies delivered through an acknowledgement callback instead of events. |
|
|
128
|
+
| Python: implement `AgentAdapter` (`send`, `describe`, enter/exit). | |
|
|
129
|
+
|
|
130
|
+
## Quickstart (no agent needed)
|
|
131
|
+
|
|
132
|
+
From a clone, in a virtualenv:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
136
|
+
pip install -e ".[http]"
|
|
137
|
+
askback eval
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
That last command grades the built-in demo agent against the bundled example suite:
|
|
141
|
+
|
|
142
|
+
```
|
|
143
|
+
No --config given: grading the built-in demo agent against the bundled example suite. This is a
|
|
144
|
+
self-check, not a test of your agent. Point it at yours with --config.
|
|
145
|
+
...
|
|
146
|
+
8/8 passed (100%) mean score 1.00 critical failures 0
|
|
147
|
+
PASSED: no critical checks failed.
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Exit `0`. It works from a plain `pip install` too: the demo agent and its suite ship in the
|
|
151
|
+
package. Now see it **fail**. In one terminal, start a toy agent with a deliberate flaw:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
python examples/toy-agent/agent.py
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
In another, grade it:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
askback --config examples/toy-agent/config.yaml eval
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
It exits `1`: asked about a parking policy that does not exist, the toy agent invented one.
|
|
164
|
+
Restart it with `--fixed` and the same command exits `0`. The [quickstart](docs/quickstart.md)
|
|
165
|
+
goes on to save a baseline and catch a regression.
|
|
166
|
+
|
|
167
|
+
## Point it at YOUR agent
|
|
168
|
+
|
|
169
|
+
1. **Install the transport you need.** `pip install -e ".[http]"` or `".[socketio]"`.
|
|
170
|
+
2. **Start from the template**, [`config/askback.example.yaml`](config/askback.example.yaml), and
|
|
171
|
+
edit its `agent:` block. For HTTP:
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
agent:
|
|
175
|
+
adapter: http
|
|
176
|
+
url: https://support.example.com/api/chat
|
|
177
|
+
request_template: {message: "{message}", conversation: "{session_id}"}
|
|
178
|
+
response_mapping: {text: answer, status: outcome, route: agent.name, citations: sources}
|
|
179
|
+
principal_headers:
|
|
180
|
+
default: {Authorization: "Bearer ${AGENT_TOKEN}"} # unset variable = refused, by name
|
|
181
|
+
side_effecting_tools: [issue_refund, cancel_order, send_email]
|
|
182
|
+
|
|
183
|
+
evaluation:
|
|
184
|
+
dataset_dir: suites
|
|
185
|
+
reports_dir: reports
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
`{message}` and `{session_id}` are the only placeholders. `response_mapping` takes dotted
|
|
189
|
+
paths (`choices.0.message.content`); only `text` is required. For Socket.IO, copy
|
|
190
|
+
[`examples/socketio-agent/config.yaml`](examples/socketio-agent/config.yaml): you name the
|
|
191
|
+
events that carry text, metadata, the terminator and errors, plus the login hook and rate
|
|
192
|
+
limits. Secrets come from the environment (`${VAR}`), never from the file. askback does not
|
|
193
|
+
read a `.env` file: export the variables.
|
|
194
|
+
3. **Write a first suite** in `suites/` ([format below](#the-test-case-format)). Start with 3
|
|
195
|
+
cases: one fact it must state, one thing it must say "I don't know" to, one multi-turn.
|
|
196
|
+
`src/askback/demo/suites/demo.yaml` is a small suite to copy from.
|
|
197
|
+
4. **Run it**, and open the report:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
askback --config my-config.yaml eval
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Reports land in `evaluation.reports_dir` as JSON (canonical) and HTML. Exit `0` passed,
|
|
204
|
+
`1` a case or gate failed, `2` regression against the baseline, `3` the run could not be
|
|
205
|
+
judged (bad config, missing file).
|
|
206
|
+
5. **Read the failures before the pass rate.** Every verdict lists the named checks behind it.
|
|
207
|
+
|
|
208
|
+
A multi-turn case only works if your agent keeps history keyed on the session id the
|
|
209
|
+
framework sends. If it does not, the failures will be about memory and will be real.
|
|
210
|
+
|
|
211
|
+
### When you don't control the agent
|
|
212
|
+
|
|
213
|
+
Often the agent runs on someone else's machine, behind a VPN. Send them
|
|
214
|
+
**[`tools/probe_agent.py`](tools/probe_agent.py)**: one standalone file that imports nothing
|
|
215
|
+
from this repository.
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
pip install "python-socketio[client]" # only for Socket.IO; "websockets>=14" for plain WebSocket
|
|
219
|
+
python probe_agent.py http://HOST:PORT "a question the agent can answer" --socketio --namespace /chat
|
|
220
|
+
python probe_agent.py http://HOST:PORT/api/chat "a question the agent can answer"
|
|
221
|
+
python probe_agent.py http://HOST:PORT "a question..." --discover # HTTP: try common paths
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
It tries the common request shapes (and, for Socket.IO, the common event names), records
|
|
225
|
+
**every event the server sends, by name**, and writes `probe-result.json`. They review it and
|
|
226
|
+
send it back: no repository, no credentials, no access granted. It contains the agent's reply,
|
|
227
|
+
which may be internal content, so they should read it first.
|
|
228
|
+
|
|
229
|
+
Its "draft event classification" is a guess to be corrected, not trusted. Against our own
|
|
230
|
+
example server it picks the wrong content event. The transcript is the useful part.
|
|
231
|
+
|
|
232
|
+
There is no separate questionnaire in this repository. These are the facts a working config
|
|
233
|
+
needs, and what to ask for if the probe cannot get them:
|
|
234
|
+
|
|
235
|
+
| Fact | Why |
|
|
236
|
+
|---|---|
|
|
237
|
+
| Transport, URL, namespace | which adapter |
|
|
238
|
+
| Request shape, or event name and payload | `request_template` |
|
|
239
|
+
| Where the reply text, status, route, citations, retrieved chunks and tool calls live | `response_mapping`, and which evaluators can run at all |
|
|
240
|
+
| How a reply ends (terminator event, or a quiet period) | Socket.IO `terminator` / `idle_timeout_s`; there is no default |
|
|
241
|
+
| Whether history is kept per session id or per connection | multi-turn |
|
|
242
|
+
| Auth: static token, or sign-in (OTP?) with refresh; token lifetime | `principal_headers` or the `auth` hook |
|
|
243
|
+
| Two or more test identities with different entitlements | isolation suites |
|
|
244
|
+
| Rate limits | `rate_limit` |
|
|
245
|
+
| Slowest observed reply | `timeout_s` (default 120) |
|
|
246
|
+
| Which tools act on the world | `side_effecting_tools` |
|
|
247
|
+
| The structure of the source document, if it cites one | the citation manifest |
|
|
248
|
+
|
|
249
|
+
## The test case format
|
|
250
|
+
|
|
251
|
+
A case is a conversation: a list of turns, each with its own expectations. Files are YAML or
|
|
252
|
+
JSON (multi-turn); CSV and XLSX hold one turn per row. Unknown keys are refused, because a
|
|
253
|
+
misspelt expectation is a check that silently never runs.
|
|
254
|
+
|
|
255
|
+
**An order bot** (schema version 3, which adds `clarification` and `persona`):
|
|
256
|
+
|
|
257
|
+
```yaml
|
|
258
|
+
schema_version: 3
|
|
259
|
+
|
|
260
|
+
cases:
|
|
261
|
+
- id: return-window
|
|
262
|
+
description: A fact the bot must state, and a wrong answer it must not give.
|
|
263
|
+
tags: [policy]
|
|
264
|
+
turns:
|
|
265
|
+
- user: Can I return the headphones I bought 40 days ago?
|
|
266
|
+
expect:
|
|
267
|
+
expected_status: answered
|
|
268
|
+
required_facts: ["30 days"]
|
|
269
|
+
prohibited_content: ["yes, you can return"]
|
|
270
|
+
|
|
271
|
+
- id: refund-asks-which-order
|
|
272
|
+
description: A vague request. The bot should ask which order; we answer it, and grade the reply.
|
|
273
|
+
turns:
|
|
274
|
+
- user: I want my money back
|
|
275
|
+
clarification:
|
|
276
|
+
policy: require # asking is the right behaviour here
|
|
277
|
+
max_rounds: 1
|
|
278
|
+
answers: ["Order 1182, the blue kettle"]
|
|
279
|
+
expect:
|
|
280
|
+
expected_status: answered
|
|
281
|
+
required_facts: ["1182"]
|
|
282
|
+
|
|
283
|
+
- id: status-check-never-refunds
|
|
284
|
+
description: Looking is not acting.
|
|
285
|
+
risk: high
|
|
286
|
+
turns:
|
|
287
|
+
- user: Where is my order 1182?
|
|
288
|
+
expect:
|
|
289
|
+
required_facts: ["1182"]
|
|
290
|
+
forbidden_tools: [issue_refund, cancel_order]
|
|
291
|
+
|
|
292
|
+
- id: remembers-the-order
|
|
293
|
+
description: Turn 3 must still know what turn 1 said.
|
|
294
|
+
turns:
|
|
295
|
+
- user: My order number is 1182.
|
|
296
|
+
evaluate: false # setup: sent, never graded
|
|
297
|
+
- user: What colour was the thing I ordered?
|
|
298
|
+
expect: {expected_status: answered}
|
|
299
|
+
- user: Which order number did I give you?
|
|
300
|
+
expect:
|
|
301
|
+
required_facts: ["1182"]
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
**The same shape for the HR assistant that ships as the worked example** (schema version 2
|
|
305
|
+
still loads unchanged):
|
|
306
|
+
|
|
307
|
+
```yaml
|
|
308
|
+
schema_version: 2
|
|
309
|
+
|
|
310
|
+
cases:
|
|
311
|
+
- id: leave-entitlement
|
|
312
|
+
tags: [policy, leave]
|
|
313
|
+
turns:
|
|
314
|
+
- user: How many days of annual leave do I get?
|
|
315
|
+
expect:
|
|
316
|
+
expected_route: policy_agent
|
|
317
|
+
expected_status: answered
|
|
318
|
+
expected_docs: [HR-001]
|
|
319
|
+
required_facts: ["24 days"]
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
Other expectations: `expected_intent`, `expected_tools`, `min_citations`, `reference_answer`
|
|
323
|
+
(for the judge), `must_refuse`, `expected_handoff`, `verify_citations`. Case-level: `samples`
|
|
324
|
+
and `min_passing_samples` (run N times, require K), `user_id`/`tenant_id`, `risk`, `tags`,
|
|
325
|
+
`persona`. Full reference: [Writing your first suite](docs/first-suite.md).
|
|
326
|
+
|
|
327
|
+
### Clarifying questions
|
|
328
|
+
|
|
329
|
+
Every reply is classified as a question or an answer by a free, deterministic detector. A
|
|
330
|
+
question is answered (your scripted `answers`, then an optional model-backed `persona` that
|
|
331
|
+
may only state facts from a sheet you give it) up to `max_rounds`, and the turn is graded on
|
|
332
|
+
the **last** reply. Two outcomes are kept apart on purpose:
|
|
333
|
+
|
|
334
|
+
| It ends... | Meaning | Reported as |
|
|
335
|
+
|---|---|---|
|
|
336
|
+
| still asking after `max_rounds` answers | the agent did not converge | an **error** |
|
|
337
|
+
| asking, and we had nothing to say | a gap in **our suite** | **unmeasured**: cannot pass, counted separately, the message names the fix |
|
|
338
|
+
|
|
339
|
+
The detector shows its score and the signals that fired in the report, so a misfire is
|
|
340
|
+
visible. A reply that invites a query is not treated as a clarification when the user's
|
|
341
|
+
message was only a greeting. It is tuned on a handful of real replies and has run on little
|
|
342
|
+
else: see the limitations.
|
|
343
|
+
|
|
344
|
+
### Citations
|
|
345
|
+
|
|
346
|
+
`expect.verify_citations: true` resolves the chapters, pages and document names in the
|
|
347
|
+
reply's prose against a manifest you keep by hand (roughly the contents page):
|
|
348
|
+
|
|
349
|
+
```yaml
|
|
350
|
+
name: Harborview Staff Handbook 2025
|
|
351
|
+
aliases: [Harborview Staff Handbook, the staff handbook]
|
|
352
|
+
total_pages: 300
|
|
353
|
+
chapters:
|
|
354
|
+
- {number: 5, title: "Workplace Safety, Conduct & Corrective Action", pages: [28, 45]}
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
Set `evaluation.citations.manifest` to its path. With no manifest, or no citation it can
|
|
358
|
+
read, the turn is skipped and says so. A citation that does not resolve is CRITICAL.
|
|
359
|
+
|
|
360
|
+
## What the evaluators are
|
|
361
|
+
|
|
362
|
+
| Module | Checks |
|
|
363
|
+
|---|---|
|
|
364
|
+
| `rule_based` | route, intent, status, tools, required facts, prohibited content, citation count, schema |
|
|
365
|
+
| `rag` | retrieval recall, rank (MRR), citation support (against what the agent itself returned), groundedness |
|
|
366
|
+
| `safety` | refusal behaviour, injection resistance, secret leakage, tenant isolation, tool misuse |
|
|
367
|
+
| `pii_egress` | email, phone, national id, card, IBAN. `salary_figure` is opt-in. Always on. |
|
|
368
|
+
| `escalation` | an escalated turn carries a ticket reference or queue name |
|
|
369
|
+
| `citation_existence` | cited chapters, pages and documents exist in the manifest |
|
|
370
|
+
| `llm_judge` | correctness, completeness, groundedness, reasoning, clarity. Free heuristic by default; a model is optional and metered, and can be one you run yourself. |
|
|
371
|
+
|
|
372
|
+
A case passes when its weighted score clears the threshold **and** no critical check failed.
|
|
373
|
+
One failing critical check fails the run at any pass rate. A judge that falls back to the
|
|
374
|
+
heuristic is marked degraded and never blended into the mean.
|
|
375
|
+
|
|
376
|
+
### Gates and commands
|
|
377
|
+
|
|
378
|
+
```bash
|
|
379
|
+
askback eval # no --config: the built-in demo (green)
|
|
380
|
+
askback --config my.yaml eval --suite orders # chosen suites (repeatable)
|
|
381
|
+
askback --config my.yaml eval --tag policy # cases with a tag
|
|
382
|
+
askback --config my.yaml eval --case return-window
|
|
383
|
+
askback --config my.yaml eval --compare-baseline
|
|
384
|
+
askback --config my.yaml baseline save # promote the newest run to the baseline
|
|
385
|
+
askback --config my.yaml calibrate # measure the judge against human labels
|
|
386
|
+
askback --version
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
`--config` goes before the subcommand. A path that does not exist is an error, never a quiet
|
|
390
|
+
fallback. Per-suite floors (`evaluation.suite_must_pass`) are checked independently of the
|
|
391
|
+
global rate; none are set by default, and a floor on a suite that did not run is an error, not
|
|
392
|
+
a pass.
|
|
393
|
+
|
|
394
|
+
## The judge, and where your data goes
|
|
395
|
+
|
|
396
|
+
With no judge configured, a free heuristic grades (a lexical proxy). A model judge is
|
|
397
|
+
**metered** and sees each graded question, reference answer and reply. The first option in the
|
|
398
|
+
config template is a model you run yourself (vLLM, Ollama, LiteLLM, a company gateway): it needs
|
|
399
|
+
no API key, and nothing leaves your network.
|
|
400
|
+
|
|
401
|
+
```yaml
|
|
402
|
+
evaluation:
|
|
403
|
+
judge:
|
|
404
|
+
llm: {provider: openai, base_url: "http://localhost:11434/v1", model: llama3}
|
|
405
|
+
```
|
|
406
|
+
|
|
407
|
+
OpenAI, Azure OpenAI and Anthropic are supported too. Anthropic has no default model: you name
|
|
408
|
+
it. The judge has its own `JUDGE_`-prefixed credentials and never reads the agent's.
|
|
409
|
+
[docs/cost.md](docs/cost.md) has the controls: a spend ceiling, a cache that stores scores only,
|
|
410
|
+
and a degradation gate.
|
|
411
|
+
|
|
412
|
+
## Reports and CI
|
|
413
|
+
|
|
414
|
+
**Reports are redacted before they are written.** `ci` (default) replaces PII and credential
|
|
415
|
+
matches, cuts replies to 200 characters and withholds retrieved passages; `metrics_only` keeps
|
|
416
|
+
verdicts and scores and no text; `full` is for your own machine and is refused when `CI` is
|
|
417
|
+
set. `ci` does **not** promise that a 200-character excerpt is free of internal content: for
|
|
418
|
+
anything where the answer itself is sensitive, use `metrics_only`. The JSON is canonical; HTML,
|
|
419
|
+
Excel and markdown are rendered from it. `reports/` is gitignored.
|
|
420
|
+
|
|
421
|
+
**CI.** [`.github/workflows/evaluate.yml`](.github/workflows/evaluate.yml) is the template:
|
|
422
|
+
lint, types and tests on every pull request; the built-in example suite as the PR gate (green on
|
|
423
|
+
a clean checkout); the HR simulator as a separate, **non-gating** demonstration that is expected
|
|
424
|
+
to be red; nothing for forks (no secrets, never `pull_request_target`); and a nightly run with a
|
|
425
|
+
model judge under a spending ceiling. Actions are pinned by commit SHA and dev tools by
|
|
426
|
+
`constraints.txt`.
|
|
427
|
+
|
|
428
|
+
## Known limitations
|
|
429
|
+
|
|
430
|
+
- **Never validated against a real agent.** See Status. Treat first runs as configuration
|
|
431
|
+
work, not as results.
|
|
432
|
+
- **Defaults tuned on HR text.** The judge threshold (0.55), the pass threshold, the
|
|
433
|
+
clarification detector's weights and greeting vocabulary, and the citation resolver's idea of
|
|
434
|
+
a "document" were developed against an HR simulator and a few real replies. They have run on
|
|
435
|
+
no other domain here. Calibrate before trusting a threshold on your agent.
|
|
436
|
+
- **Greeting recognition misses.** On 20 messages written before it was measured, it read 17
|
|
437
|
+
correctly. The misses were greetings read as requests ("Good evening, hope you're doing
|
|
438
|
+
well"), which is the harmless direction; no real request was read as a greeting (0 of 10).
|
|
439
|
+
- **Citation checks prove existence, not support.** They also read only "Chapter N", "page N"
|
|
440
|
+
and document-looking names, attribute a page to a chapter conservatively, handle one
|
|
441
|
+
document, and take a claimed chapter title only when it is title-cased, so a manifest whose
|
|
442
|
+
titles are in sentence case has its titles left unchecked rather than guessed at.
|
|
443
|
+
- **Where a reply wraps does not change its grade.** Every phrase pattern matches across a
|
|
444
|
+
line break, hard wraps are undone before citations are read, and a fixture re-wrapped at 32,
|
|
445
|
+
40, 60 and 80 columns is graded identically (`tests/test_wrapping.py`).
|
|
446
|
+
- **Transports**: see the table above. HTTP retries can deliver a turn twice (set
|
|
447
|
+
`max_attempts: 1` for an agent that is not idempotent per session); the Socket.IO adapter
|
|
448
|
+
never re-sends a turn, and a lost connection ends that session.
|
|
449
|
+
- **The PII gate is on for every turn and cannot be switched off per case.** Which categories
|
|
450
|
+
it checks is configurable (`evaluator_options.pii_egress.categories`). `salary_figure` is off
|
|
451
|
+
by default because it reads "you paid $1,299" as a salary; turn it on if your agent talks
|
|
452
|
+
about pay. Allowlist regexes must match the whole detected value.
|
|
453
|
+
- **The HR simulator is not in the wheel.** `askback_sim` is a source-checkout fixture. The simulator commands (`ask`, `chat`, `info`, `serve`)
|
|
454
|
+
and the HR example need `pip install -e .` from a clone.
|
|
455
|
+
- **No suite floors, no default suites.** You set them. The HR example sets its own.
|
|
456
|
+
|
|
457
|
+
## How it works
|
|
458
|
+
|
|
459
|
+
```
|
|
460
|
+
suite files --> loader --> TestRunner --> AgentAdapter --> your agent
|
|
461
|
+
yaml json | (HTTP, Socket.IO, built-in demo, or your own)
|
|
462
|
+
csv xlsx v
|
|
463
|
+
EvaluationEngine
|
|
464
|
+
rule_based rag safety pii_egress escalation
|
|
465
|
+
citation_existence llm_judge
|
|
466
|
+
|
|
|
467
|
+
RunReport --> redaction --> JSON (canonical) --> HTML, xlsx, markdown
|
|
468
|
+
\--> gates --> exit code \--> baseline comparison
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
Everything crossing a boundary is a `typing.Protocol`: an adapter or a judge implements a
|
|
472
|
+
shape and inherits nothing from the framework.
|
|
473
|
+
|
|
474
|
+
```
|
|
475
|
+
src/askback/ the framework (core, datasets, adapters, evaluation, runner, reporting, providers, demo, api)
|
|
476
|
+
src/askback_sim/ the HR example's simulator (source checkout only; not in the wheel)
|
|
477
|
+
config/ askback.example.yaml, the template to copy
|
|
478
|
+
tools/ probe_agent.py, the standalone wire-format probe
|
|
479
|
+
examples/hr/ the HR example: simulator config, 101 cases, corpus, baseline (red on purpose)
|
|
480
|
+
examples/ toy HTTP agent, Socket.IO agent (with login hook), a Python adapter, a custom evaluator
|
|
481
|
+
docs/ quickstart, first suite, adapters, evaluators, calibration, cost, the HR audit, backlog
|
|
482
|
+
```
|
|
483
|
+
|
|
484
|
+
## Development
|
|
485
|
+
|
|
486
|
+
```bash
|
|
487
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
488
|
+
pip install -c constraints.txt -e ".[all,dev]"
|
|
489
|
+
ruff check src tests tools examples && mypy src && pytest --strict-env
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
`mypy src` is strict across every module with no exemption list. A plain `pytest` skips test
|
|
493
|
+
modules whose optional dependency is missing and prints a `MODULES NOT TESTED` block;
|
|
494
|
+
`--strict-env`, which CI uses, makes that an error. Extras: `http`, `socketio`,
|
|
495
|
+
`socketio-server` (for the example server's tests), `probe`, `excel`, `server`, `llm`, `all`.
|
|
496
|
+
|
|
497
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the invariants a change must not break,
|
|
498
|
+
[CHANGELOG.md](CHANGELOG.md), [SECURITY.md](SECURITY.md) and the
|
|
499
|
+
[code of conduct](CODE_OF_CONDUCT.md).
|
|
500
|
+
|
|
501
|
+
## Licence
|
|
502
|
+
|
|
503
|
+
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). The corpus, suites, fixtures and
|
|
504
|
+
calibration records in this repository are invented.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
askback/__init__.py,sha256=-mvWr9mI6NK1Yhu3Nd_Gy8OeOrXRl-m4sJ-eD5tf35g,891
|
|
2
|
+
askback/__main__.py,sha256=YX0iP2d4nBZTedK-V7oamOYXUw-h7dUKfNERCqYebTU,197
|
|
3
|
+
askback/cli.py,sha256=eH4cceY_PvvEo3TEETIf4RABL0R9OraTls4IlkVf1Bg,24889
|
|
4
|
+
askback/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
askback/adapters/__init__.py,sha256=ttA4RfvqkSXpkLdVTQfUwoRx7vxvdCFWl7gcnBywtrw,325
|
|
6
|
+
askback/adapters/base.py,sha256=MlIUaIftqWq0UhbsRDFlKILPIdLYY60ZYGAAuaroJqw,7485
|
|
7
|
+
askback/adapters/factory.py,sha256=lUSFHGTe7yYB_zowbHKh2K6ounr5kXaqcv1dFPFiTEw,6880
|
|
8
|
+
askback/adapters/http.py,sha256=nwrfVEGF2O3G7zw_pK7ayY6o8jfxKs7dBU7Lmhm1Z2Q,18971
|
|
9
|
+
askback/adapters/in_process.py,sha256=5ykMmcQLWmWZCjf2CNptRWukKwr0Q5_9cZjuGoR01aI,5133
|
|
10
|
+
askback/adapters/payload.py,sha256=5vqIK7U-N4U8urBnSdk81zq1mExeT6Gr9npPvaUDBzk,7781
|
|
11
|
+
askback/adapters/scripted.py,sha256=kSS7Z9kF17CgfW4ckD9bOosqfi6HVyWDfSGS_3c2ULw,4749
|
|
12
|
+
askback/adapters/socketio_adapter.py,sha256=LDP_u3l_gh3A2K3bSm6cMj9D4O1WoKkY5skCOG7VP7Y,38524
|
|
13
|
+
askback/api/__init__.py,sha256=rjbP8UTAApHaJEvafcubdX34yOfB5EphUuUH5u_GupM,402
|
|
14
|
+
askback/api/corpus_store.py,sha256=NzD3gVCLx2v5OCQn03SUgr_uUQTP4_obL80V5w_pAzI,3077
|
|
15
|
+
askback/api/dashboard.py,sha256=_2atfPOHtSSdOGFpuBYUPTQfVlybAdXlQBHPsKntIQs,18274
|
|
16
|
+
askback/api/server.py,sha256=P9EP-JCMt6cqOA3RX_FQI2VDXLXlQPz5PXeJk1nH9gw,7731
|
|
17
|
+
askback/api/state.py,sha256=2G_imtSjaOSbqvME7Y4h7DHiyflZxP-7KX_Ua5ISZuI,2942
|
|
18
|
+
askback/api/static/app.css,sha256=Cl23wr5N7HuFoz1QbZRoYheGFXFxjHajcbXOn9fX8iM,20440
|
|
19
|
+
askback/api/static/app.js,sha256=qALxQBnZSHHjBIkWaOUo_C52Vz1RUvIK65CBfOUXYPU,3386
|
|
20
|
+
askback/api/static/core.js,sha256=f4bLnwEyGRqV-oby5DOrfbFmLLayYnSc3mSG1OCJK4M,7592
|
|
21
|
+
askback/api/static/index.html,sha256=G-Jj8FSZHVLswceg07T330iNOgW04z8S5ex4G_L2wm0,6512
|
|
22
|
+
askback/api/static/panels.js,sha256=8yX6AxVXJS-bnRXXGMQwCGDJh1z8xpn-PPdUa2XD5Ts,26630
|
|
23
|
+
askback/core/__init__.py,sha256=ri2imxYhh6Vjl8HzlHcl-T93d5FGOgCMdOpPyOEPomo,1115
|
|
24
|
+
askback/core/config.py,sha256=hgGfAOfsdU3WodaWChVIfyFgpWJ0bwpG8ZpSoBRQGmc,8407
|
|
25
|
+
askback/core/contracts.py,sha256=-FppoSL94lx525urXSKDcV0nJP2sSiWwpEZf1OEB2Qc,6060
|
|
26
|
+
askback/core/errors.py,sha256=i6BYNE5FSEXgxiZ25GuyiSTGZZcTaWMzdq3Ap82dWX0,1063
|
|
27
|
+
askback/core/logging.py,sha256=2Gr0PxQxoCh_svt2w5p-ySa03TCl_kTZjC034Nykh_8,1755
|
|
28
|
+
askback/core/registry.py,sha256=cAP4sCS6IC3__kB7Udt6p5JWrGZEK0sI0S3fqxFmz30,2335
|
|
29
|
+
askback/core/text.py,sha256=CWczFu22rvtsmOB8K3JHFkQHZ8IycktxV54QUj_9mGI,6389
|
|
30
|
+
askback/datasets/__init__.py,sha256=hfj3eREJme2hqghlQM58xkZhXUcsFmgPF0ijcqpGVyA,332
|
|
31
|
+
askback/datasets/loader.py,sha256=upoTr91OHOaQwuq9aPi73ii7NbvPsFEQtdK1KEaecL0,10199
|
|
32
|
+
askback/datasets/schema.py,sha256=DNR_k7Pg8nGW1NxKjEsX7qJ_2_qtZUerpvZQRowaUE0,22728
|
|
33
|
+
askback/demo/__init__.py,sha256=09JwBIsvps1dkQp6T9mX-v2PUuK4PU1R3RQOzGqbAC8,1535
|
|
34
|
+
askback/demo/agent.py,sha256=vLK4VY3uIHA8NbK-OTA2v_X4itePxnPNfHYKoTuRO4U,8094
|
|
35
|
+
askback/demo/suites/demo.yaml,sha256=w5Oh98Gg270PmRjO0gy4ps_-rN3SzqS7PTm0gjVWVuY,3691
|
|
36
|
+
askback/evaluation/__init__.py,sha256=-gnSH1CRTCt3GzVafh8nHaqRrGsovP_cjVirlG3SrRU,2240
|
|
37
|
+
askback/evaluation/base.py,sha256=HuRJ_jYSgg2cvmkhatX8mrWZHcaAqeR7faQKloYTg5Y,7379
|
|
38
|
+
askback/evaluation/calibration.py,sha256=Uyfv2gHhdCD-GblopTodsJcPQSJ5rHQ-uqjO7xrnmUY,15632
|
|
39
|
+
askback/evaluation/citation_eval.py,sha256=0gvANUvCnLzhoCtNqhaNzfO3LkWyNbc6BpCZrtwgu0Q,24550
|
|
40
|
+
askback/evaluation/clarification.py,sha256=VKq6vY9YVlHw-6ritCISg3HIzD9q8QROU9i1o1V4Wak,20718
|
|
41
|
+
askback/evaluation/engine.py,sha256=S5iPLpC9iUMNXaHrB8RrgGkSmfsg4Zv5NyjOh5y6Nr0,8274
|
|
42
|
+
askback/evaluation/escalation_eval.py,sha256=bFTH2W0lpFQUuy22eX0GpKoE1bnT-yNmebOGef6cgYk,4675
|
|
43
|
+
askback/evaluation/gates.py,sha256=o3gs5MubAY2a6q_nvddS7fvYRyJIK7rX8yFnj27JwPw,12245
|
|
44
|
+
askback/evaluation/judge_cache.py,sha256=JL09FxgA2-Y94zRzk32D3OelJPHrtMWIu1wli3ygrNQ,4105
|
|
45
|
+
askback/evaluation/llm_judge.py,sha256=NrVxVCHoFhHv3E9dpMCBOs4Ox2yNjUE0mUkuXEX521A,19418
|
|
46
|
+
askback/evaluation/patterns.py,sha256=wRtFbpSg2Lrx59B0smWzs3Lwif5ImqiUCITxIdIevBM,2701
|
|
47
|
+
askback/evaluation/pii_eval.py,sha256=dVveJxj8N5Bt8i2npKQ-2pbWUnAEpBfRWiBPWXWBWE4,10476
|
|
48
|
+
askback/evaluation/rag_eval.py,sha256=fbrF_DIqA8tL_9RSSD8ipPgTrKDJ1vv22Prs8WwiyYg,6812
|
|
49
|
+
askback/evaluation/regression.py,sha256=tvdrdU07ihN5rbBTGNqQap_5XQEXu7ANbomNzSbVANI,16985
|
|
50
|
+
askback/evaluation/results.py,sha256=Ja5kFnqjFNqA6IaAtOomTHpmt5B21zzKHr_fiROKcPc,24411
|
|
51
|
+
askback/evaluation/rule_based.py,sha256=0zVgrYdv0WtGhXZec8oLKgnhdxN7wsaUMsPhicsbXRU,8201
|
|
52
|
+
askback/evaluation/safety_eval.py,sha256=954T8PFPCvDekKGLRB0_7ZJLmB4l6dQbBPLHCt_-jwI,6835
|
|
53
|
+
askback/providers/__init__.py,sha256=VYr8QA5c0Guzgovp9lG5uEdEcjUFTk1JLqDExbScfHY,373
|
|
54
|
+
askback/providers/base.py,sha256=RLa5WrjQF3YWqcFwK2q1kKbsTT-dURnzZIRY9FVcaaU,5591
|
|
55
|
+
askback/providers/openai_compat.py,sha256=RQq37rq98PSUJ7dMx8uLulNjcELC3jCBIJ4eGfdM4i4,12377
|
|
56
|
+
askback/reporting/__init__.py,sha256=8wIID3wklVeToy96gA1jgfZ1soQzvv2DERQILIs2tlc,2551
|
|
57
|
+
askback/reporting/excel_report.py,sha256=m_lqqKcglId2MOnxrvX1mf6npnSXUrQ2ZIqJAzzgv1c,4622
|
|
58
|
+
askback/reporting/html_report.py,sha256=SZ1OQuqJFd9FIe1L0gokTbBQCkQhJ5xIZJKtEVHZOJY,12533
|
|
59
|
+
askback/reporting/json_report.py,sha256=-kPodqRx-94Kh0EAJOV1OnG7bWmEKvkOiKDEozLE2OA,866
|
|
60
|
+
askback/reporting/markdown_report.py,sha256=ZUsJIAP34dB3LgSDYMnYn5rqQ82rloIkmNWpRzg111Y,3714
|
|
61
|
+
askback/reporting/redaction.py,sha256=wGeT5R-PhPEpbjDTU2Nf_0Q9X9st0qaQXEbFTNhiVEA,8493
|
|
62
|
+
askback/runner/__init__.py,sha256=UgwHuhH_2bRUMXOrfWWn-EZ0wwYikHqbRX_lKFfKrBs,85
|
|
63
|
+
askback/runner/responders.py,sha256=F2s8aT3FbzYaam-oe-xi38NxcCMe6HTnoL9nwTU9ooM,12266
|
|
64
|
+
askback/runner/test_runner.py,sha256=580TS3TrFo6PqcXVeuWeQ23evE9n5SdAKooF13XhYSo,18779
|
|
65
|
+
askback-0.1.0.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
|
|
66
|
+
askback-0.1.0.dist-info/licenses/NOTICE,sha256=XMOtjgzPXu7ZpZa6xKqd_25Ykb9RmXyJU8SZMiUnVSY,818
|
|
67
|
+
askback-0.1.0.dist-info/METADATA,sha256=kDV0ZAh1aSBhFw-JEZOgTWwUs9qDXIDWPlP2PgOEMRk,27049
|
|
68
|
+
askback-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
69
|
+
askback-0.1.0.dist-info/entry_points.txt,sha256=9xj5MNjh3riLH9a90_ftTubxNhkHhzxuIe7RmWHirxo,45
|
|
70
|
+
askback-0.1.0.dist-info/top_level.txt,sha256=dOKhojt4x-FCAloCgXc3HX0jaGSyxXVHg8z54oApmkA,8
|
|
71
|
+
askback-0.1.0.dist-info/RECORD,,
|