askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/* Shared helpers: DOM building, the API client, formatting and toasts.
|
|
2
|
+
*
|
|
3
|
+
* Loaded first; panels.js and app.js depend on everything exported here onto
|
|
4
|
+
* `window`. Plain ES2020, no modules, no build step - the file is served as-is
|
|
5
|
+
* from the package, including inside a container with no network.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/* ─────────────────────────────────────────────────────────── DOM building */
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Build an element.
|
|
12
|
+
*
|
|
13
|
+
* el('div', { class: 'card' }, el('span', {}, 'hi'))
|
|
14
|
+
*
|
|
15
|
+
* Children that are strings become text nodes, never markup. Every value shown
|
|
16
|
+
* in this dashboard comes from the corpus, a model reply or a test case - i.e.
|
|
17
|
+
* data we do not control - so there is deliberately no innerHTML path here.
|
|
18
|
+
*/
|
|
19
|
+
function el(tag, props, ...children) {
|
|
20
|
+
const node = document.createElement(tag);
|
|
21
|
+
|
|
22
|
+
for (const [key, value] of Object.entries(props || {})) {
|
|
23
|
+
if (value === null || value === undefined || value === false) continue;
|
|
24
|
+
if (key === 'class') node.className = value;
|
|
25
|
+
else if (key === 'dataset') Object.assign(node.dataset, value);
|
|
26
|
+
else if (key === 'style') Object.assign(node.style, value);
|
|
27
|
+
else if (key.startsWith('on') && typeof value === 'function') {
|
|
28
|
+
node.addEventListener(key.slice(2).toLowerCase(), value);
|
|
29
|
+
} else if (key in node && key !== 'list') {
|
|
30
|
+
node[key] = value;
|
|
31
|
+
} else {
|
|
32
|
+
node.setAttribute(key, value);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
for (const child of children.flat()) {
|
|
37
|
+
if (child === null || child === undefined || child === false) continue;
|
|
38
|
+
node.append(child instanceof Node ? child : document.createTextNode(String(child)));
|
|
39
|
+
}
|
|
40
|
+
return node;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Replace an element's children in one shot. */
|
|
44
|
+
function mount(target, ...children) {
|
|
45
|
+
const node = typeof target === 'string' ? document.getElementById(target) : target;
|
|
46
|
+
node.replaceChildren(...children.flat().filter((c) => c !== null && c !== undefined && c !== false));
|
|
47
|
+
return node;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const $ = (id) => document.getElementById(id);
|
|
51
|
+
|
|
52
|
+
/* ────────────────────────────────────────────────────────────── API client */
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Every call goes through here so error handling is uniform: FastAPI reports
|
|
56
|
+
* problems as `{detail: ...}`, and the caller should see that message rather
|
|
57
|
+
* than a bare "500".
|
|
58
|
+
*/
|
|
59
|
+
async function request(method, path, body) {
|
|
60
|
+
const options = { method, headers: {} };
|
|
61
|
+
if (body !== undefined) {
|
|
62
|
+
options.headers['content-type'] = 'application/json';
|
|
63
|
+
options.body = JSON.stringify(body);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let response;
|
|
67
|
+
try {
|
|
68
|
+
response = await fetch(path, options);
|
|
69
|
+
} catch (cause) {
|
|
70
|
+
throw new Error(`Cannot reach the server. Is it still running? (${cause.message})`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const text = await response.text();
|
|
74
|
+
let payload = null;
|
|
75
|
+
try {
|
|
76
|
+
payload = text ? JSON.parse(text) : null;
|
|
77
|
+
} catch {
|
|
78
|
+
payload = text;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (!response.ok) {
|
|
82
|
+
const detail = payload && payload.detail ? payload.detail : payload || response.statusText;
|
|
83
|
+
throw new Error(typeof detail === 'string' ? detail : JSON.stringify(detail));
|
|
84
|
+
}
|
|
85
|
+
return payload;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const api = {
|
|
89
|
+
get: (path, params) => {
|
|
90
|
+
const query = params ? `?${new URLSearchParams(params)}` : '';
|
|
91
|
+
return request('GET', path + query);
|
|
92
|
+
},
|
|
93
|
+
post: (path, body) => request('POST', path, body),
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
/* ──────────────────────────────────────────────────────────── formatting */
|
|
97
|
+
|
|
98
|
+
const fmt = {
|
|
99
|
+
pct: (value) => `${Math.round((value || 0) * 100)}%`,
|
|
100
|
+
score: (value) => (value === null || value === undefined ? '–' : Number(value).toFixed(2)),
|
|
101
|
+
ms: (value) => `${Math.round(value || 0)} ms`,
|
|
102
|
+
|
|
103
|
+
/** Report timestamps are ISO strings; show them in the reader's locale. */
|
|
104
|
+
when: (iso) => {
|
|
105
|
+
if (!iso) return '';
|
|
106
|
+
const date = new Date(iso);
|
|
107
|
+
return Number.isNaN(date.getTime()) ? iso : date.toLocaleString();
|
|
108
|
+
},
|
|
109
|
+
|
|
110
|
+
truncate: (text, limit) => {
|
|
111
|
+
const value = String(text ?? '');
|
|
112
|
+
return value.length > limit ? `${value.slice(0, limit - 1)}…` : value;
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/* ─────────────────────────────────────────────────────────────── feedback */
|
|
117
|
+
|
|
118
|
+
/** Transient message, bottom right. `kind` is 'info' | 'success' | 'error'. */
|
|
119
|
+
function toast(message, kind = 'info') {
|
|
120
|
+
const node = el('div', { class: `toast ${kind}` }, message);
|
|
121
|
+
$('toasts').append(node);
|
|
122
|
+
// Errors linger: they usually need reading and often need acting on.
|
|
123
|
+
setTimeout(() => node.remove(), kind === 'error' ? 7000 : 3200);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Wrap an async handler so a rejected promise surfaces instead of vanishing. */
|
|
127
|
+
function guard(fn) {
|
|
128
|
+
return async (...args) => {
|
|
129
|
+
try {
|
|
130
|
+
return await fn(...args);
|
|
131
|
+
} catch (error) {
|
|
132
|
+
toast(error.message, 'error');
|
|
133
|
+
console.error(error);
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Disable a button while its action runs, so it cannot be double-fired. */
|
|
140
|
+
async function withBusy(button, label, action) {
|
|
141
|
+
const original = button.textContent;
|
|
142
|
+
button.disabled = true;
|
|
143
|
+
button.textContent = label;
|
|
144
|
+
try {
|
|
145
|
+
return await action();
|
|
146
|
+
} finally {
|
|
147
|
+
button.disabled = false;
|
|
148
|
+
button.textContent = original;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/* ────────────────────────────────────────────────── shared view fragments */
|
|
153
|
+
|
|
154
|
+
/** A labelled statistic. Used for both the inspector and the eval summary. */
|
|
155
|
+
function statCard(label, value, tone = '', className = 'card') {
|
|
156
|
+
return el('div', { class: className },
|
|
157
|
+
el('div', { class: 'k' }, label),
|
|
158
|
+
el('div', { class: `v ${tone}` }, value));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** A horizontal score bar with a label and a numeric readout. */
|
|
162
|
+
function scoreBar(label, value, { max = 1, winner = false, dim = false } = {}) {
|
|
163
|
+
const ratio = max > 0 ? Math.max(0, Math.min(1, value / max)) : 0;
|
|
164
|
+
return el('div', {},
|
|
165
|
+
el('div', { class: `bar-row ${winner ? 'winner' : ''}` },
|
|
166
|
+
el('span', { class: 'bar-label', title: label }, label),
|
|
167
|
+
el('span', { class: 'bar-value' }, fmt.score(value))),
|
|
168
|
+
el('div', { class: 'bar-track' },
|
|
169
|
+
el('div', { class: `bar-fill ${dim ? 'dim' : ''}`, style: { width: `${ratio * 100}%` } })));
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** A key/value line for the inspector. */
|
|
173
|
+
function kvRow(key, value) {
|
|
174
|
+
return el('div', { class: 'kv' },
|
|
175
|
+
el('span', { class: 'k' }, key),
|
|
176
|
+
el('span', { class: 'v' }, value));
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Build a table from a header list and an array of cell arrays. */
|
|
180
|
+
function dataTable(headers, rows, { numeric = [] } = {}) {
|
|
181
|
+
return el('table', {},
|
|
182
|
+
el('thead', {}, el('tr', {}, headers.map((h, i) =>
|
|
183
|
+
el('th', { class: numeric.includes(i) ? 'num' : '' }, h)))),
|
|
184
|
+
el('tbody', {}, rows.map((row) => el('tr', {}, row.map((cell, i) =>
|
|
185
|
+
el('td', { class: numeric.includes(i) ? 'num' : '' },
|
|
186
|
+
cell instanceof Node ? cell : String(cell ?? '')))))));
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function emptyState(title, body) {
|
|
190
|
+
return el('div', { class: 'empty-state' }, el('h3', {}, title), el('p', {}, body));
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function passPill(passed, label) {
|
|
194
|
+
return el('span', { class: `pill ${passed ? 'pass' : 'fail'}` }, label || (passed ? 'pass' : 'fail'));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/* Exported onto window for panels.js and app.js. */
|
|
198
|
+
Object.assign(window, {
|
|
199
|
+
el, mount, $, api, fmt, toast, guard, withBusy,
|
|
200
|
+
statCard, scoreBar, kvRow, dataTable, emptyState, passPill,
|
|
201
|
+
});
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>askback dashboard</title>
|
|
7
|
+
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>🤖</text></svg>">
|
|
8
|
+
<link rel="stylesheet" href="/static/app.css">
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
|
|
12
|
+
<!-- Panels are plain sections; app.js shows one at a time. No framework, no
|
|
13
|
+
build step: this has to work from a container with no network. -->
|
|
14
|
+
|
|
15
|
+
<header class="topbar">
|
|
16
|
+
<div class="brand">
|
|
17
|
+
<span class="brand-mark" aria-hidden="true"></span>
|
|
18
|
+
<span class="brand-name">askback</span>
|
|
19
|
+
<span class="brand-sub">dashboard</span>
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
<nav class="tabs" id="tabs" role="tablist">
|
|
23
|
+
<button class="tab" role="tab" data-panel="console">Console</button>
|
|
24
|
+
<button class="tab" role="tab" data-panel="evaluation">Evaluation</button>
|
|
25
|
+
<button class="tab" role="tab" data-panel="knowledge">Knowledge</button>
|
|
26
|
+
<button class="tab" role="tab" data-panel="routing">Routing</button>
|
|
27
|
+
</nav>
|
|
28
|
+
|
|
29
|
+
<div class="topbar-right">
|
|
30
|
+
<div class="engine-chip" id="engine-chip" title="Engine wiring">
|
|
31
|
+
<span class="dot" id="engine-dot"></span>
|
|
32
|
+
<span id="engine-summary">connecting…</span>
|
|
33
|
+
</div>
|
|
34
|
+
<button class="icon-btn" id="reload-engine" title="Rebuild the engine from the files on disk">Reload</button>
|
|
35
|
+
<button class="icon-btn" id="theme-toggle" title="Switch light / dark" aria-label="Switch theme">◐</button>
|
|
36
|
+
</div>
|
|
37
|
+
</header>
|
|
38
|
+
|
|
39
|
+
<main>
|
|
40
|
+
|
|
41
|
+
<!-- ─────────────────────────────────────────────────────────────── console -->
|
|
42
|
+
<section class="panel" id="panel-console">
|
|
43
|
+
<div class="split">
|
|
44
|
+
|
|
45
|
+
<div class="chat-column">
|
|
46
|
+
<div class="chat-toolbar">
|
|
47
|
+
<label>as user
|
|
48
|
+
<input id="chat-user" value="dashboard" size="10">
|
|
49
|
+
</label>
|
|
50
|
+
<label>tenant
|
|
51
|
+
<select id="chat-tenant">
|
|
52
|
+
<option value="default">default</option>
|
|
53
|
+
<option value="acme">acme</option>
|
|
54
|
+
</select>
|
|
55
|
+
</label>
|
|
56
|
+
<button class="ghost-btn" id="chat-reset">Clear</button>
|
|
57
|
+
</div>
|
|
58
|
+
|
|
59
|
+
<div class="chat-log" id="chat-log" aria-live="polite"></div>
|
|
60
|
+
|
|
61
|
+
<div class="suggestions" id="suggestions"></div>
|
|
62
|
+
|
|
63
|
+
<form class="composer" id="composer">
|
|
64
|
+
<textarea id="chat-input" rows="1" placeholder="Ask something… (Enter to send, Shift+Enter for a new line)"></textarea>
|
|
65
|
+
<button class="primary-btn" type="submit" id="chat-send">Send</button>
|
|
66
|
+
</form>
|
|
67
|
+
</div>
|
|
68
|
+
|
|
69
|
+
<aside class="inspector" id="inspector">
|
|
70
|
+
<div class="empty-state">
|
|
71
|
+
<h3>Inspector</h3>
|
|
72
|
+
<p>Ask a question, or click any earlier reply, to see how the engine reached it: routing scores, retrieved passages, citations, tool calls and the safety verdict.</p>
|
|
73
|
+
</div>
|
|
74
|
+
</aside>
|
|
75
|
+
|
|
76
|
+
</div>
|
|
77
|
+
</section>
|
|
78
|
+
|
|
79
|
+
<!-- ────────────────────────────────────────────────────────────── evaluation -->
|
|
80
|
+
<section class="panel" id="panel-evaluation">
|
|
81
|
+
<div class="eval-layout">
|
|
82
|
+
|
|
83
|
+
<aside class="runs-rail">
|
|
84
|
+
<div class="rail-head">
|
|
85
|
+
<h3>Runs</h3>
|
|
86
|
+
<button class="ghost-btn" id="runs-refresh" title="Re-read the reports directory">↻</button>
|
|
87
|
+
</div>
|
|
88
|
+
<div class="runs-list" id="runs-list"></div>
|
|
89
|
+
</aside>
|
|
90
|
+
|
|
91
|
+
<div class="eval-main">
|
|
92
|
+
<div class="run-controls">
|
|
93
|
+
<div class="control-group">
|
|
94
|
+
<span class="control-label">Suites</span>
|
|
95
|
+
<div class="chip-row" id="suite-chips"></div>
|
|
96
|
+
</div>
|
|
97
|
+
<div class="control-group">
|
|
98
|
+
<span class="control-label">Tags</span>
|
|
99
|
+
<div class="chip-row" id="tag-chips"></div>
|
|
100
|
+
</div>
|
|
101
|
+
<div class="control-actions">
|
|
102
|
+
<input id="run-name" placeholder="run name (optional)" size="18">
|
|
103
|
+
<button class="primary-btn" id="run-eval">Run evaluation</button>
|
|
104
|
+
</div>
|
|
105
|
+
</div>
|
|
106
|
+
|
|
107
|
+
<div id="eval-body">
|
|
108
|
+
<div class="empty-state">
|
|
109
|
+
<h3>No run loaded</h3>
|
|
110
|
+
<p>Pick a suite and run an evaluation, or select a past run from the left.</p>
|
|
111
|
+
</div>
|
|
112
|
+
</div>
|
|
113
|
+
</div>
|
|
114
|
+
|
|
115
|
+
</div>
|
|
116
|
+
</section>
|
|
117
|
+
|
|
118
|
+
<!-- ─────────────────────────────────────────────────────────────── knowledge -->
|
|
119
|
+
<section class="panel" id="panel-knowledge">
|
|
120
|
+
<div class="editor-layout">
|
|
121
|
+
|
|
122
|
+
<aside class="doc-rail">
|
|
123
|
+
<div class="rail-head">
|
|
124
|
+
<h3>Documents</h3>
|
|
125
|
+
</div>
|
|
126
|
+
<input class="rail-filter" id="doc-filter" placeholder="Filter by id, title or domain…">
|
|
127
|
+
<div class="doc-list" id="doc-list"></div>
|
|
128
|
+
<div class="rail-foot" id="corpus-stats"></div>
|
|
129
|
+
</aside>
|
|
130
|
+
|
|
131
|
+
<div class="doc-editor" id="doc-editor">
|
|
132
|
+
<div class="empty-state">
|
|
133
|
+
<h3>Knowledge base</h3>
|
|
134
|
+
<p>Select a document to read it. The corpus is read-only here: edit the JSONL on disk and press Reload, and the next question sees the change.</p>
|
|
135
|
+
</div>
|
|
136
|
+
</div>
|
|
137
|
+
|
|
138
|
+
</div>
|
|
139
|
+
</section>
|
|
140
|
+
|
|
141
|
+
<!-- ───────────────────────────────────────────────────────────────── routing -->
|
|
142
|
+
<section class="panel" id="panel-routing">
|
|
143
|
+
<div class="routing-layout">
|
|
144
|
+
|
|
145
|
+
<div class="routing-main">
|
|
146
|
+
<div class="section-head">
|
|
147
|
+
<h3>Intent catalogue</h3>
|
|
148
|
+
<div class="view-switch" id="intents-view-switch">
|
|
149
|
+
<button class="seg active" data-view="table">Table</button>
|
|
150
|
+
<button class="seg" data-view="yaml">YAML</button>
|
|
151
|
+
</div>
|
|
152
|
+
</div>
|
|
153
|
+
<div id="intents-body"></div>
|
|
154
|
+
</div>
|
|
155
|
+
|
|
156
|
+
<aside class="route-tester">
|
|
157
|
+
<h3>Route tester</h3>
|
|
158
|
+
<p class="muted">Classify without answering. The fast loop when tuning keywords.</p>
|
|
159
|
+
<input id="route-input" placeholder="Type a question…">
|
|
160
|
+
<div id="route-result"></div>
|
|
161
|
+
</aside>
|
|
162
|
+
|
|
163
|
+
</div>
|
|
164
|
+
</section>
|
|
165
|
+
|
|
166
|
+
</main>
|
|
167
|
+
|
|
168
|
+
<div class="toasts" id="toasts" aria-live="polite"></div>
|
|
169
|
+
|
|
170
|
+
<!-- Load order matters: core defines the helpers the other two use. -->
|
|
171
|
+
<script src="/static/core.js"></script>
|
|
172
|
+
<script src="/static/panels.js"></script>
|
|
173
|
+
<script src="/static/app.js"></script>
|
|
174
|
+
</body>
|
|
175
|
+
</html>
|