askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
askback/api/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""HTTP API. Imported lazily so the core never requires FastAPI.
|
|
2
|
+
|
|
3
|
+
from askback.api import create_app
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def create_app(config: Any = None) -> Any:
|
|
12
|
+
"""Build the FastAPI application (requires `pip install -e .[server]`)."""
|
|
13
|
+
from .server import create_app as _create_app
|
|
14
|
+
|
|
15
|
+
return _create_app(config)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = ["create_app"]
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""File-aware read access to the JSONL corpus, for the dashboard's Knowledge panel.
|
|
2
|
+
|
|
3
|
+
`askback_sim.retrieval.corpus` loads documents for *searching* and deliberately
|
|
4
|
+
forgets which file each came from. The panel shows that, so rather than
|
|
5
|
+
complicate the retrieval path with bookkeeping it does not need, this module
|
|
6
|
+
reads the files directly.
|
|
7
|
+
|
|
8
|
+
There is no write path, on purpose. The corpus is the thing every answer depends
|
|
9
|
+
on, and it is edited as files, under review, not from a web panel.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ..core.errors import DatasetError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class CorpusEntry:
|
|
24
|
+
"""One document plus the file it lives in."""
|
|
25
|
+
|
|
26
|
+
doc_id: str
|
|
27
|
+
record: dict[str, Any]
|
|
28
|
+
file: Path
|
|
29
|
+
|
|
30
|
+
def summary(self) -> dict[str, Any]:
|
|
31
|
+
"""The compact form the dashboard list view needs."""
|
|
32
|
+
return {
|
|
33
|
+
"doc_id": self.doc_id,
|
|
34
|
+
"title": str(self.record.get("title", "")),
|
|
35
|
+
"source": str(self.record.get("source", "")),
|
|
36
|
+
"metadata": dict(self.record.get("metadata") or {}),
|
|
37
|
+
"characters": len(str(self.record.get("text", ""))),
|
|
38
|
+
"file": self.file.name,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class CorpusStore:
|
|
43
|
+
"""The .jsonl files in the corpus directory, read as documents."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, corpus_path: str | Path) -> None:
|
|
46
|
+
self.root = Path(corpus_path)
|
|
47
|
+
|
|
48
|
+
def files(self) -> list[Path]:
|
|
49
|
+
if self.root.is_file():
|
|
50
|
+
return [self.root]
|
|
51
|
+
return sorted(self.root.glob("*.jsonl")) if self.root.is_dir() else []
|
|
52
|
+
|
|
53
|
+
def load(self) -> list[CorpusEntry]:
|
|
54
|
+
"""Every document, in file order. Raises on malformed or duplicated ids."""
|
|
55
|
+
entries: list[CorpusEntry] = []
|
|
56
|
+
seen: set[str] = set()
|
|
57
|
+
|
|
58
|
+
for file in self.files():
|
|
59
|
+
for line_no, line in enumerate(file.read_text(encoding="utf-8").splitlines(), 1):
|
|
60
|
+
stripped = line.strip()
|
|
61
|
+
if not stripped or stripped.startswith("//"):
|
|
62
|
+
continue
|
|
63
|
+
try:
|
|
64
|
+
record = json.loads(stripped)
|
|
65
|
+
except json.JSONDecodeError as exc:
|
|
66
|
+
raise DatasetError(f"{file.name}:{line_no} is not valid JSON: {exc}") from exc
|
|
67
|
+
|
|
68
|
+
doc_id = str(record.get("doc_id") or "").strip()
|
|
69
|
+
if not doc_id:
|
|
70
|
+
raise DatasetError(f"{file.name}:{line_no} has no doc_id.")
|
|
71
|
+
if doc_id in seen:
|
|
72
|
+
raise DatasetError(f"{file.name}:{line_no} duplicates doc_id {doc_id!r}.")
|
|
73
|
+
seen.add(doc_id)
|
|
74
|
+
entries.append(CorpusEntry(doc_id=doc_id, record=record, file=file))
|
|
75
|
+
|
|
76
|
+
return entries
|
|
77
|
+
|
|
78
|
+
def list(self) -> list[dict[str, Any]]:
|
|
79
|
+
return [entry.summary() for entry in self.load()]
|
|
80
|
+
|
|
81
|
+
def get(self, doc_id: str) -> dict[str, Any] | None:
|
|
82
|
+
for entry in self.load():
|
|
83
|
+
if entry.doc_id == doc_id:
|
|
84
|
+
return {**entry.record, "_file": entry.file.name}
|
|
85
|
+
return None
|
askback/api/dashboard.py
ADDED
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
"""The dashboard API: everything the built-in UI needs that the public API does not.
|
|
2
|
+
|
|
3
|
+
Kept separate from `server.py` on purpose. `/chat` is a stable contract other
|
|
4
|
+
systems integrate against; these `/api/*` routes are developer tooling and are
|
|
5
|
+
free to change shape as the UI does.
|
|
6
|
+
|
|
7
|
+
Four groups, matching the four panels:
|
|
8
|
+
|
|
9
|
+
/api/console ask a question and see why the engine answered that way
|
|
10
|
+
/api/eval run suites, browse past runs, compare against the baseline
|
|
11
|
+
/api/corpus read the knowledge base
|
|
12
|
+
/api/routing read the intent catalogue
|
|
13
|
+
|
|
14
|
+
The corpus and the intent catalogue are read-only here, with no switch that makes
|
|
15
|
+
them otherwise: they decide what the agent answers, and a route that rewrites them
|
|
16
|
+
is live on every instance anyone can reach. Edit the files and reload.
|
|
17
|
+
|
|
18
|
+
Nor can a run be promoted to the baseline from here. The baseline is what every
|
|
19
|
+
later regression comparison trusts, and a reachable instance that could replace it
|
|
20
|
+
would make those comparisons lie with nothing looking wrong afterwards. That is
|
|
21
|
+
`askback baseline save`, on the CLI, where it leaves a file to commit.
|
|
22
|
+
|
|
23
|
+
The one route that changes server state is `POST /api/system/reload`, and
|
|
24
|
+
`server.allow_runtime_reload` (default false) is what permits it.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import json
|
|
30
|
+
import re
|
|
31
|
+
from collections.abc import Callable
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
from fastapi import APIRouter, Depends, HTTPException, status
|
|
36
|
+
from pydantic import BaseModel, Field
|
|
37
|
+
|
|
38
|
+
from ..adapters.factory import resolve_principals
|
|
39
|
+
from ..core.config import AppConfig, default_samples, resolve_path
|
|
40
|
+
from ..core.contracts import ChatRequest
|
|
41
|
+
from ..core.logging import get_logger
|
|
42
|
+
from ..datasets import filter_cases, load_cases
|
|
43
|
+
from ..evaluation.engine import EvaluationEngine
|
|
44
|
+
from ..evaluation.regression import Baseline, RegressionComparator
|
|
45
|
+
from ..evaluation.results import RunReport
|
|
46
|
+
from ..reporting import redacted, resolve_mode, write_reports
|
|
47
|
+
from ..runner import TestRunner
|
|
48
|
+
from .corpus_store import CorpusStore
|
|
49
|
+
from .state import EngineHolder
|
|
50
|
+
|
|
51
|
+
log = get_logger(__name__)
|
|
52
|
+
|
|
53
|
+
#: Report and run names become file paths, so they are restricted to characters
|
|
54
|
+
#: that cannot escape the reports directory.
|
|
55
|
+
_SAFE_NAME = re.compile(r"^[A-Za-z0-9._-]{1,80}$")
|
|
56
|
+
|
|
57
|
+
#: Sentinel accepted wherever a run name is expected, meaning "the saved baseline".
|
|
58
|
+
BASELINE_ALIAS = "baseline"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# --------------------------------------------------------------------------- #
|
|
62
|
+
# Request models
|
|
63
|
+
# --------------------------------------------------------------------------- #
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ConsoleChatIn(BaseModel):
|
|
67
|
+
message: str = Field(..., min_length=1, max_length=4000)
|
|
68
|
+
conversation_id: str | None = Field(None, max_length=128)
|
|
69
|
+
user_id: str = Field("dashboard", max_length=128)
|
|
70
|
+
tenant_id: str = Field("default", max_length=128)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class EvalRunIn(BaseModel):
|
|
74
|
+
suites: list[str] = Field(default_factory=list)
|
|
75
|
+
tags: list[str] = Field(default_factory=list)
|
|
76
|
+
case_ids: list[str] = Field(default_factory=list)
|
|
77
|
+
name: str | None = Field(None, max_length=80)
|
|
78
|
+
workers: int | None = Field(None, ge=1, le=32)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# --------------------------------------------------------------------------- #
|
|
82
|
+
# Router
|
|
83
|
+
# --------------------------------------------------------------------------- #
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def build_dashboard_router(
|
|
87
|
+
holder: EngineHolder,
|
|
88
|
+
cfg: AppConfig,
|
|
89
|
+
require_api_key: Callable[..., Any],
|
|
90
|
+
) -> APIRouter:
|
|
91
|
+
"""Assemble the dashboard routes around a live engine."""
|
|
92
|
+
router = APIRouter(prefix="/api", dependencies=[Depends(require_api_key)])
|
|
93
|
+
|
|
94
|
+
evaluation = cfg.evaluation
|
|
95
|
+
reports_dir = resolve_path(evaluation.get("reports_dir", "reports"))
|
|
96
|
+
dataset_dir = resolve_path(evaluation.get("dataset_dir", "suites"))
|
|
97
|
+
baseline_path = resolve_path(
|
|
98
|
+
evaluation.get("baseline_path", "baselines/baseline.json")
|
|
99
|
+
)
|
|
100
|
+
corpus = CorpusStore(resolve_path(cfg.retrieval.get("corpus_path", "examples/hr/corpus")))
|
|
101
|
+
intents_path = resolve_path(cfg.routing.get("intents_path", "examples/hr/intents.yaml"))
|
|
102
|
+
reload_allowed = bool(cfg.server.get("allow_runtime_reload", False))
|
|
103
|
+
|
|
104
|
+
def require_reload_allowed() -> None:
|
|
105
|
+
if not reload_allowed:
|
|
106
|
+
raise HTTPException(
|
|
107
|
+
status_code=status.HTTP_403_FORBIDDEN,
|
|
108
|
+
detail="Runtime reload is disabled. Set server.allow_runtime_reload to true to allow it.",
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
# ------------------------------------------------------------- console #
|
|
112
|
+
|
|
113
|
+
@router.post("/console/chat", tags=["console"])
|
|
114
|
+
def console_chat(payload: ConsoleChatIn) -> dict[str, Any]:
|
|
115
|
+
"""Answer a message and return everything behind the answer.
|
|
116
|
+
|
|
117
|
+
The public `/chat` returns what an integrator needs; this returns what a
|
|
118
|
+
tester needs, which is mostly the reasoning that led to the reply.
|
|
119
|
+
"""
|
|
120
|
+
session: dict[str, Any] = (
|
|
121
|
+
{"conversation_id": payload.conversation_id} if payload.conversation_id else {}
|
|
122
|
+
)
|
|
123
|
+
request = ChatRequest(
|
|
124
|
+
message=payload.message,
|
|
125
|
+
user_id=payload.user_id,
|
|
126
|
+
tenant_id=payload.tenant_id,
|
|
127
|
+
**session,
|
|
128
|
+
)
|
|
129
|
+
response = holder.engine.chat(request)
|
|
130
|
+
return {
|
|
131
|
+
"reply": response.reply,
|
|
132
|
+
"conversation_id": response.conversation_id,
|
|
133
|
+
"route": response.route,
|
|
134
|
+
"status": response.status.value,
|
|
135
|
+
"latency_ms": response.latency_ms,
|
|
136
|
+
"trace_id": response.trace_id,
|
|
137
|
+
"model": response.model,
|
|
138
|
+
"error": response.error,
|
|
139
|
+
"intent": {
|
|
140
|
+
"name": response.intent.name,
|
|
141
|
+
"confidence": round(response.intent.confidence, 4),
|
|
142
|
+
"rationale": response.intent.rationale,
|
|
143
|
+
# Sorted so the UI can render a ranked "why this route" list.
|
|
144
|
+
"scores": dict(
|
|
145
|
+
sorted(response.intent.scores.items(), key=lambda kv: kv[1], reverse=True)
|
|
146
|
+
),
|
|
147
|
+
},
|
|
148
|
+
"citations": [
|
|
149
|
+
{"doc_id": c.doc_id, "title": c.title, "source": c.source, "snippet": c.snippet}
|
|
150
|
+
for c in response.citations
|
|
151
|
+
],
|
|
152
|
+
"retrieved": [
|
|
153
|
+
{
|
|
154
|
+
"doc_id": c.doc_id,
|
|
155
|
+
"chunk_id": c.chunk_id,
|
|
156
|
+
"title": c.title,
|
|
157
|
+
"score": c.score,
|
|
158
|
+
"text": c.text,
|
|
159
|
+
"metadata": c.metadata,
|
|
160
|
+
}
|
|
161
|
+
for c in response.retrieved
|
|
162
|
+
],
|
|
163
|
+
"tool_calls": [
|
|
164
|
+
{
|
|
165
|
+
"name": t.name,
|
|
166
|
+
"arguments": t.arguments,
|
|
167
|
+
"ok": t.ok,
|
|
168
|
+
"error": t.error,
|
|
169
|
+
"latency_ms": t.latency_ms,
|
|
170
|
+
}
|
|
171
|
+
for t in response.tool_calls
|
|
172
|
+
],
|
|
173
|
+
"safety": {
|
|
174
|
+
"blocked": response.safety.blocked,
|
|
175
|
+
"injection_detected": response.safety.injection_detected,
|
|
176
|
+
"risk_score": response.safety.risk_score,
|
|
177
|
+
"reasons": response.safety.reasons,
|
|
178
|
+
"redactions": response.safety.redactions,
|
|
179
|
+
},
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
@router.post("/console/reset", tags=["console"])
|
|
183
|
+
def console_reset(conversation_id: str | None = None) -> dict[str, str]:
|
|
184
|
+
holder.engine.reset(conversation_id)
|
|
185
|
+
return {"status": "reset", "conversation_id": conversation_id or "all"}
|
|
186
|
+
|
|
187
|
+
@router.get("/console/route-test", tags=["console"])
|
|
188
|
+
def route_test(q: str) -> dict[str, Any]:
|
|
189
|
+
"""Classify without answering. The fast loop when tuning intents."""
|
|
190
|
+
intent, route = holder.engine.frontline.route(q)
|
|
191
|
+
return {
|
|
192
|
+
"query": q,
|
|
193
|
+
"route": route,
|
|
194
|
+
"intent": intent.name,
|
|
195
|
+
"confidence": round(intent.confidence, 4),
|
|
196
|
+
"rationale": intent.rationale,
|
|
197
|
+
"scores": dict(sorted(intent.scores.items(), key=lambda kv: kv[1], reverse=True)),
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
@router.get("/console/search", tags=["console"])
|
|
201
|
+
def search(q: str, top_k: int = 5, tenant: str = "default") -> dict[str, Any]:
|
|
202
|
+
"""Retrieval only, with no agent or generation in the way."""
|
|
203
|
+
hits = holder.engine.retriever.search(q, top_k=min(top_k, 20), tenant=tenant)
|
|
204
|
+
return {
|
|
205
|
+
"query": q,
|
|
206
|
+
"hits": [
|
|
207
|
+
{
|
|
208
|
+
"doc_id": h.doc_id,
|
|
209
|
+
"chunk_id": h.chunk_id,
|
|
210
|
+
"title": h.title,
|
|
211
|
+
"score": h.score,
|
|
212
|
+
"text": h.text,
|
|
213
|
+
}
|
|
214
|
+
for h in hits
|
|
215
|
+
],
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
# ---------------------------------------------------------- evaluation #
|
|
219
|
+
|
|
220
|
+
@router.get("/eval/cases", tags=["evaluation"])
|
|
221
|
+
def list_cases(suite: str | None = None) -> dict[str, Any]:
|
|
222
|
+
"""The test cases on disk, so the UI can show what a run will cover."""
|
|
223
|
+
cases = load_cases(dataset_dir)
|
|
224
|
+
selected = filter_cases(cases, suites=[suite] if suite else None)
|
|
225
|
+
suites: dict[str, int] = {}
|
|
226
|
+
tags: dict[str, int] = {}
|
|
227
|
+
for case in cases:
|
|
228
|
+
suites[case.suite] = suites.get(case.suite, 0) + 1
|
|
229
|
+
for tag in case.tags:
|
|
230
|
+
tags[tag] = tags.get(tag, 0) + 1
|
|
231
|
+
|
|
232
|
+
return {
|
|
233
|
+
"suites": dict(sorted(suites.items())),
|
|
234
|
+
"tags": dict(sorted(tags.items())),
|
|
235
|
+
"total": len(cases),
|
|
236
|
+
"cases": [
|
|
237
|
+
{
|
|
238
|
+
"id": c.id,
|
|
239
|
+
"suite": c.suite,
|
|
240
|
+
"message": c.turns[0].user,
|
|
241
|
+
# The fields below are the opening turn's; this says when
|
|
242
|
+
# that is not the whole case.
|
|
243
|
+
"turns": len(c.turns),
|
|
244
|
+
"description": c.description,
|
|
245
|
+
"expected_route": c.turns[0].expect.expected_route,
|
|
246
|
+
"expected_status": c.turns[0].expect.expected_status,
|
|
247
|
+
"expected_docs": c.turns[0].expect.expected_docs,
|
|
248
|
+
"required_facts": c.turns[0].expect.required_facts,
|
|
249
|
+
"must_refuse": c.turns[0].expect.must_refuse,
|
|
250
|
+
"risk": c.risk,
|
|
251
|
+
"tags": c.tags,
|
|
252
|
+
}
|
|
253
|
+
for c in selected
|
|
254
|
+
],
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
@router.post("/eval/run", tags=["evaluation"])
|
|
258
|
+
def run_eval(payload: EvalRunIn) -> dict[str, Any]:
|
|
259
|
+
"""Run the selected cases and write the reports.
|
|
260
|
+
|
|
261
|
+
Synchronous. With the offline provider a full run is well under a
|
|
262
|
+
second; once a remote model is wired in expect this to take minutes, at
|
|
263
|
+
which point move it to a background task.
|
|
264
|
+
"""
|
|
265
|
+
cases = filter_cases(
|
|
266
|
+
load_cases(
|
|
267
|
+
dataset_dir,
|
|
268
|
+
principals=resolve_principals(cfg),
|
|
269
|
+
default_samples=default_samples(evaluation),
|
|
270
|
+
),
|
|
271
|
+
suites=payload.suites or None,
|
|
272
|
+
tags=payload.tags or None,
|
|
273
|
+
ids=payload.case_ids or None,
|
|
274
|
+
)
|
|
275
|
+
if not cases:
|
|
276
|
+
raise HTTPException(
|
|
277
|
+
status_code=status.HTTP_400_BAD_REQUEST,
|
|
278
|
+
detail="No test cases matched that selection.",
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
name = safe_name(payload.name) if payload.name else None
|
|
282
|
+
runner = TestRunner(
|
|
283
|
+
holder.adapter,
|
|
284
|
+
EvaluationEngine.from_config(evaluation, safety=cfg.safety),
|
|
285
|
+
workers=int(payload.workers or evaluation.get("parallel_workers", 4)),
|
|
286
|
+
)
|
|
287
|
+
report = runner.run(cases, run_id=name)
|
|
288
|
+
# The response is a report leaving the machine like any other, so it is
|
|
289
|
+
# the redacted one that was written, not the run as the runner holds it.
|
|
290
|
+
redaction = resolve_mode(evaluation)
|
|
291
|
+
written = write_reports(
|
|
292
|
+
report, reports_dir, formats=("json", "html"), stem=report.run_id, redaction=redaction
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
return {
|
|
296
|
+
**redacted(report, redaction).to_dict(),
|
|
297
|
+
"artifacts": {fmt: str(path) for fmt, path in written.items()},
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
@router.get("/eval/runs", tags=["evaluation"])
|
|
301
|
+
def list_runs() -> dict[str, Any]:
|
|
302
|
+
"""Past runs, newest first, summarised from the JSON reports on disk."""
|
|
303
|
+
runs: list[dict[str, Any]] = []
|
|
304
|
+
if reports_dir.is_dir():
|
|
305
|
+
for path in sorted(
|
|
306
|
+
reports_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True
|
|
307
|
+
):
|
|
308
|
+
try:
|
|
309
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
310
|
+
except (OSError, json.JSONDecodeError):
|
|
311
|
+
continue # a half-written report should not break the list
|
|
312
|
+
runs.append(
|
|
313
|
+
{
|
|
314
|
+
"name": path.stem,
|
|
315
|
+
"run_id": data.get("run_id", path.stem),
|
|
316
|
+
"finished_at": data.get("finished_at", ""),
|
|
317
|
+
"summary": data.get("summary", {}),
|
|
318
|
+
"context": data.get("context", {}),
|
|
319
|
+
"has_html": (reports_dir / f"{path.stem}.html").exists(),
|
|
320
|
+
}
|
|
321
|
+
)
|
|
322
|
+
return {"runs": runs, "baseline_exists": baseline_path.exists()}
|
|
323
|
+
|
|
324
|
+
@router.get("/eval/runs/{name}", tags=["evaluation"])
|
|
325
|
+
def get_run(name: str) -> dict[str, Any]:
|
|
326
|
+
return _load_report_dict(name)
|
|
327
|
+
|
|
328
|
+
@router.get("/eval/compare", tags=["evaluation"])
|
|
329
|
+
def compare(candidate: str, baseline: str = BASELINE_ALIAS) -> dict[str, Any]:
|
|
330
|
+
"""Regression comparison between two runs, or a run and the baseline."""
|
|
331
|
+
comparator = RegressionComparator(
|
|
332
|
+
tolerance=float(evaluation.get("regression_tolerance", 0.05))
|
|
333
|
+
)
|
|
334
|
+
# The saved baseline is a summary; any other name is a whole run report.
|
|
335
|
+
data = _load_report_dict(baseline)
|
|
336
|
+
result = comparator.compare(
|
|
337
|
+
Baseline.from_dict(data) if baseline == BASELINE_ALIAS else RunReport.from_dict(data),
|
|
338
|
+
RunReport.from_dict(_load_report_dict(candidate)),
|
|
339
|
+
)
|
|
340
|
+
return {"baseline": baseline, "candidate": candidate, **result.to_dict()}
|
|
341
|
+
|
|
342
|
+
# -------------------------------------------------------------- corpus #
|
|
343
|
+
|
|
344
|
+
@router.get("/corpus", tags=["corpus"])
|
|
345
|
+
def list_documents() -> dict[str, Any]:
|
|
346
|
+
documents = corpus.list()
|
|
347
|
+
return {
|
|
348
|
+
"documents": documents,
|
|
349
|
+
"files": [f.name for f in corpus.files()],
|
|
350
|
+
"total_characters": sum(d["characters"] for d in documents),
|
|
351
|
+
"indexed_chunks": len(holder.engine.retriever),
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
@router.get("/corpus/{doc_id}", tags=["corpus"])
|
|
355
|
+
def get_document(doc_id: str) -> dict[str, Any]:
|
|
356
|
+
document = corpus.get(doc_id)
|
|
357
|
+
if document is None:
|
|
358
|
+
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No document {doc_id!r}.")
|
|
359
|
+
return document
|
|
360
|
+
|
|
361
|
+
# ------------------------------------------------------------- routing #
|
|
362
|
+
|
|
363
|
+
@router.get("/routing/intents", tags=["routing"])
|
|
364
|
+
def get_intents() -> dict[str, Any]:
|
|
365
|
+
"""The raw YAML, plus the parsed view the UI renders as a table."""
|
|
366
|
+
text = intents_path.read_text(encoding="utf-8")
|
|
367
|
+
return {
|
|
368
|
+
"yaml": text,
|
|
369
|
+
"path": str(intents_path),
|
|
370
|
+
"intents": _parse_intents(text),
|
|
371
|
+
"agents": sorted(holder.engine.agents),
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
# -------------------------------------------------------------- system #
|
|
375
|
+
|
|
376
|
+
@router.get("/system/info", tags=["system"])
|
|
377
|
+
def system_info() -> dict[str, Any]:
|
|
378
|
+
return {**holder.describe(), "reload_allowed": reload_allowed}
|
|
379
|
+
|
|
380
|
+
@router.post("/system/reload", tags=["system"], dependencies=[Depends(require_reload_allowed)])
|
|
381
|
+
def system_reload() -> dict[str, Any]:
|
|
382
|
+
holder.reload()
|
|
383
|
+
return holder.describe()
|
|
384
|
+
|
|
385
|
+
# ------------------------------------------------------------- helpers #
|
|
386
|
+
|
|
387
|
+
def _load_report_dict(name: str) -> dict[str, Any]:
|
|
388
|
+
path = baseline_path if name == BASELINE_ALIAS else reports_dir / f"{safe_name(name)}.json"
|
|
389
|
+
if not path.exists():
|
|
390
|
+
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No report {name!r} at {path}.")
|
|
391
|
+
loaded: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
|
|
392
|
+
return loaded
|
|
393
|
+
|
|
394
|
+
return router
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# --------------------------------------------------------------------------- #
|
|
398
|
+
# Module helpers
|
|
399
|
+
# --------------------------------------------------------------------------- #
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def safe_name(name: str) -> str:
|
|
403
|
+
"""Reject anything that could escape the reports directory.
|
|
404
|
+
|
|
405
|
+
Run names arrive from the UI and become file paths, so this is the boundary
|
|
406
|
+
between a user-supplied string and the filesystem.
|
|
407
|
+
"""
|
|
408
|
+
candidate = Path(name).name # strips any directory component outright
|
|
409
|
+
if not _SAFE_NAME.match(candidate):
|
|
410
|
+
raise HTTPException(
|
|
411
|
+
status.HTTP_400_BAD_REQUEST,
|
|
412
|
+
f"Invalid name {name!r}: use letters, digits, dot, dash or underscore.",
|
|
413
|
+
)
|
|
414
|
+
return candidate
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _parse_intents(text: str) -> list[dict[str, Any]]:
|
|
418
|
+
"""Parse the intents YAML into the flat rows the UI table renders."""
|
|
419
|
+
try:
|
|
420
|
+
import yaml
|
|
421
|
+
except ImportError as exc: # pragma: no cover - environment dependent
|
|
422
|
+
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "PyYAML is not installed.") from exc
|
|
423
|
+
|
|
424
|
+
try:
|
|
425
|
+
data = yaml.safe_load(text) or {}
|
|
426
|
+
except yaml.YAMLError as exc:
|
|
427
|
+
raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Invalid YAML: {exc}") from exc
|
|
428
|
+
|
|
429
|
+
entries = data.get("intents", []) if isinstance(data, dict) else []
|
|
430
|
+
rows: list[dict[str, Any]] = []
|
|
431
|
+
for entry in entries:
|
|
432
|
+
if not isinstance(entry, dict) or "name" not in entry:
|
|
433
|
+
raise HTTPException(status.HTTP_400_BAD_REQUEST, f"Malformed intent entry: {entry!r}")
|
|
434
|
+
rows.append(
|
|
435
|
+
{
|
|
436
|
+
"name": str(entry["name"]),
|
|
437
|
+
"route": str(entry.get("route", "fallback_agent")),
|
|
438
|
+
"description": str(entry.get("description", "")),
|
|
439
|
+
"keywords": [str(k) for k in entry.get("keywords", [])],
|
|
440
|
+
"examples": [str(e) for e in entry.get("examples", [])],
|
|
441
|
+
"priority": float(entry.get("priority", 1.0)),
|
|
442
|
+
}
|
|
443
|
+
)
|
|
444
|
+
return rows
|