askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
askback/api/server.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""FastAPI application: the public API, plus the built-in dashboard.
|
|
2
|
+
|
|
3
|
+
The engine is built once at startup and reused: loading the corpus per request
|
|
4
|
+
would dominate latency. `InMemoryConversationStore` is per process, so run a
|
|
5
|
+
single worker locally, and swap in a shared `ConversationStore` before scaling
|
|
6
|
+
out (see `askback_sim.memory`).
|
|
7
|
+
|
|
8
|
+
askback serve # dashboard at /, API docs at /docs
|
|
9
|
+
curl -X POST localhost:8000/chat -H "content-type: application/json" \
|
|
10
|
+
-d '{"message": "How many days do I have to return an item?"}'
|
|
11
|
+
|
|
12
|
+
Two surfaces, deliberately separate:
|
|
13
|
+
|
|
14
|
+
/chat, /info, /search the stable contract other systems integrate against
|
|
15
|
+
/api/* developer tooling behind the dashboard, free to change
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from fastapi import Depends, FastAPI, Header, HTTPException, status
|
|
24
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
25
|
+
from fastapi.responses import FileResponse
|
|
26
|
+
from fastapi.staticfiles import StaticFiles
|
|
27
|
+
from pydantic import BaseModel, Field
|
|
28
|
+
|
|
29
|
+
from .. import __version__
|
|
30
|
+
from ..core.config import AppConfig, resolve_path
|
|
31
|
+
from ..core.contracts import ChatRequest
|
|
32
|
+
from ..core.logging import get_logger, setup_logging
|
|
33
|
+
from .dashboard import build_dashboard_router, safe_name
|
|
34
|
+
from .state import EngineHolder
|
|
35
|
+
|
|
36
|
+
log = get_logger(__name__)
|
|
37
|
+
|
|
38
|
+
STATIC_DIR = Path(__file__).parent / "static"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# --------------------------------------------------------------------------- #
|
|
42
|
+
# Wire models. Separate from the internal contracts on purpose: the HTTP surface
|
|
43
|
+
# should be free to stay stable while the internals evolve.
|
|
44
|
+
# --------------------------------------------------------------------------- #
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ChatIn(BaseModel):
|
|
48
|
+
message: str = Field(..., min_length=1, max_length=4000)
|
|
49
|
+
conversation_id: str | None = Field(None, max_length=128)
|
|
50
|
+
user_id: str = Field("api_user", max_length=128)
|
|
51
|
+
tenant_id: str = Field("default", max_length=128)
|
|
52
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class CitationOut(BaseModel):
|
|
56
|
+
doc_id: str
|
|
57
|
+
title: str = ""
|
|
58
|
+
source: str = ""
|
|
59
|
+
snippet: str = ""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ChatOut(BaseModel):
|
|
63
|
+
reply: str
|
|
64
|
+
conversation_id: str
|
|
65
|
+
route: str
|
|
66
|
+
status: str
|
|
67
|
+
intent: str
|
|
68
|
+
intent_confidence: float
|
|
69
|
+
citations: list[CitationOut] = Field(default_factory=list)
|
|
70
|
+
tools_used: list[str] = Field(default_factory=list)
|
|
71
|
+
safety_blocked: bool = False
|
|
72
|
+
latency_ms: float = 0.0
|
|
73
|
+
trace_id: str = ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ResetIn(BaseModel):
|
|
77
|
+
conversation_id: str | None = None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def create_app(config: AppConfig | None = None) -> FastAPI:
|
|
81
|
+
"""Build the application. Import-safe: nothing runs until this is called."""
|
|
82
|
+
cfg = config or AppConfig.load()
|
|
83
|
+
setup_logging(str(cfg.logging.get("level", "INFO")))
|
|
84
|
+
|
|
85
|
+
app = FastAPI(
|
|
86
|
+
title="askback dashboard",
|
|
87
|
+
version=__version__,
|
|
88
|
+
description="Configurable conversational agent with routing, RAG, safety and evaluation.",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
origins = cfg.server.get("cors_origins", ["*"])
|
|
92
|
+
app.add_middleware(
|
|
93
|
+
CORSMiddleware,
|
|
94
|
+
allow_origins=origins,
|
|
95
|
+
allow_methods=["GET", "POST"],
|
|
96
|
+
allow_headers=["*"],
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Built once, shared by every request. The holder exists so the dashboard can
|
|
100
|
+
# rebuild it after the files change on disk, without restarting the server.
|
|
101
|
+
holder = EngineHolder(cfg)
|
|
102
|
+
expected_key = str(cfg.server.get("api_key", "") or "")
|
|
103
|
+
|
|
104
|
+
def require_api_key(x_api_key: str | None = Header(default=None)) -> None:
|
|
105
|
+
"""No-op when `server.api_key` is unset, which is the local default."""
|
|
106
|
+
if expected_key and x_api_key != expected_key:
|
|
107
|
+
raise HTTPException(
|
|
108
|
+
status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or missing API key."
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
@app.get("/health", tags=["ops"])
|
|
112
|
+
def health() -> dict[str, Any]:
|
|
113
|
+
"""Liveness probe. Cheap enough for a load balancer to hit often."""
|
|
114
|
+
return {"status": "ok", "version": app.version}
|
|
115
|
+
|
|
116
|
+
@app.get("/info", tags=["ops"], dependencies=[Depends(require_api_key)])
|
|
117
|
+
def info() -> dict[str, Any]:
|
|
118
|
+
"""How the engine is wired: provider, retriever, agents, tools, guards."""
|
|
119
|
+
return holder.engine.describe()
|
|
120
|
+
|
|
121
|
+
@app.post("/chat", response_model=ChatOut, tags=["chat"], dependencies=[Depends(require_api_key)])
|
|
122
|
+
def chat(payload: ChatIn) -> ChatOut:
|
|
123
|
+
"""Answer one message."""
|
|
124
|
+
request = ChatRequest(
|
|
125
|
+
message=payload.message,
|
|
126
|
+
user_id=payload.user_id,
|
|
127
|
+
tenant_id=payload.tenant_id,
|
|
128
|
+
metadata=payload.metadata,
|
|
129
|
+
**({"conversation_id": payload.conversation_id} if payload.conversation_id else {}),
|
|
130
|
+
)
|
|
131
|
+
response = holder.engine.chat(request)
|
|
132
|
+
return ChatOut(
|
|
133
|
+
reply=response.reply,
|
|
134
|
+
conversation_id=response.conversation_id,
|
|
135
|
+
route=response.route,
|
|
136
|
+
status=response.status.value,
|
|
137
|
+
intent=response.intent.name,
|
|
138
|
+
intent_confidence=round(response.intent.confidence, 4),
|
|
139
|
+
citations=[
|
|
140
|
+
CitationOut(
|
|
141
|
+
doc_id=c.doc_id, title=c.title, source=c.source, snippet=c.snippet[:240]
|
|
142
|
+
)
|
|
143
|
+
for c in response.citations
|
|
144
|
+
],
|
|
145
|
+
tools_used=response.tool_names,
|
|
146
|
+
safety_blocked=response.safety.blocked,
|
|
147
|
+
latency_ms=response.latency_ms,
|
|
148
|
+
trace_id=response.trace_id,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
@app.post("/reset", tags=["chat"], dependencies=[Depends(require_api_key)])
|
|
152
|
+
def reset(payload: ResetIn) -> dict[str, str]:
|
|
153
|
+
"""Clear one conversation, or every conversation when no id is given."""
|
|
154
|
+
holder.engine.reset(payload.conversation_id)
|
|
155
|
+
return {"status": "reset", "conversation_id": payload.conversation_id or "all"}
|
|
156
|
+
|
|
157
|
+
@app.get("/search", tags=["debug"], dependencies=[Depends(require_api_key)])
|
|
158
|
+
def search(q: str, top_k: int = 5, tenant: str = "default") -> dict[str, Any]:
|
|
159
|
+
"""Inspect retrieval directly. Useful when an answer looks wrong."""
|
|
160
|
+
hits = holder.engine.retriever.search(q, top_k=min(top_k, 20), tenant=tenant)
|
|
161
|
+
return {
|
|
162
|
+
"query": q,
|
|
163
|
+
"hits": [
|
|
164
|
+
{"doc_id": h.doc_id, "title": h.title, "score": h.score, "text": h.text[:300]}
|
|
165
|
+
for h in hits
|
|
166
|
+
],
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
app.include_router(build_dashboard_router(holder, cfg, require_api_key))
|
|
170
|
+
_mount_dashboard(app, cfg)
|
|
171
|
+
return app
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _mount_dashboard(app: FastAPI, cfg: AppConfig) -> None:
|
|
175
|
+
"""Serve the dashboard at `/`, if its assets shipped with the package.
|
|
176
|
+
|
|
177
|
+
A slim install without the static files should still serve the API rather
|
|
178
|
+
than fail at startup, so a missing directory is a warning, not an error.
|
|
179
|
+
"""
|
|
180
|
+
if not STATIC_DIR.is_dir():
|
|
181
|
+
log.warning("Dashboard assets not found at %s; serving API only.", STATIC_DIR)
|
|
182
|
+
return
|
|
183
|
+
|
|
184
|
+
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
|
185
|
+
|
|
186
|
+
@app.get("/", include_in_schema=False)
|
|
187
|
+
def dashboard() -> FileResponse:
|
|
188
|
+
return FileResponse(STATIC_DIR / "index.html")
|
|
189
|
+
|
|
190
|
+
@app.get("/reports/{name}.html", include_in_schema=False)
|
|
191
|
+
def report_page(name: str) -> FileResponse:
|
|
192
|
+
"""Open a generated HTML report from the dashboard's run list."""
|
|
193
|
+
path = resolve_path(cfg.evaluation.get("reports_dir", "reports")) / f"{safe_name(name)}.html"
|
|
194
|
+
if not path.exists():
|
|
195
|
+
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No HTML report named {name!r}.")
|
|
196
|
+
return FileResponse(path)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# Uvicorn entry point: `uvicorn askback.api.server:app`
|
|
200
|
+
app = None # set lazily by `create_app`; `askback serve` builds it explicitly
|
askback/api/state.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Holds the live engine so the dashboard can rebuild it without a restart.
|
|
2
|
+
|
|
3
|
+
The engine is expensive to build (it indexes the whole corpus), so the service
|
|
4
|
+
builds one at startup and shares it. The corpus and the intent catalogue are
|
|
5
|
+
edited as files on disk, which means something has to swap the engine
|
|
6
|
+
afterwards; that is this.
|
|
7
|
+
|
|
8
|
+
Swapping is atomic from a request's point of view: a new engine is built first
|
|
9
|
+
and only assigned once it succeeds, so a broken edit leaves the running engine
|
|
10
|
+
untouched and the error goes back to whoever asked for the reload.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import threading
|
|
16
|
+
from typing import TYPE_CHECKING, Any
|
|
17
|
+
|
|
18
|
+
from ..adapters import AgentAdapter
|
|
19
|
+
from ..adapters.factory import build_adapter
|
|
20
|
+
from ..core.config import AppConfig
|
|
21
|
+
from ..core.logging import get_logger
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from askback_sim.engine import ConversationEngine
|
|
25
|
+
|
|
26
|
+
log = get_logger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _build_engine(config: AppConfig) -> ConversationEngine:
|
|
30
|
+
"""Build the simulator's engine, importing it only when one is built.
|
|
31
|
+
|
|
32
|
+
The adapter factory will not import the simulator, so this is where the
|
|
33
|
+
server - which is allowed to know it exists - does, and hands the engine on.
|
|
34
|
+
"""
|
|
35
|
+
from askback_sim.engine import ConversationEngine
|
|
36
|
+
|
|
37
|
+
return ConversationEngine.from_config(config)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class EngineHolder:
|
|
41
|
+
"""One mutable reference to the current engine."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, config: AppConfig) -> None:
|
|
44
|
+
self.config = config
|
|
45
|
+
self._lock = threading.Lock()
|
|
46
|
+
self._engine = _build_engine(config)
|
|
47
|
+
self._adapter = build_adapter(config, engine=self._engine)
|
|
48
|
+
self.reload_count = 0
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def engine(self) -> ConversationEngine:
|
|
52
|
+
"""The engine to serve this request with."""
|
|
53
|
+
return self._engine
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def adapter(self) -> AgentAdapter:
|
|
57
|
+
"""The current engine as the evaluation runner reaches it.
|
|
58
|
+
|
|
59
|
+
Shared by every run: adapters count their entries, so concurrent runs
|
|
60
|
+
can each hold it open.
|
|
61
|
+
"""
|
|
62
|
+
return self._adapter
|
|
63
|
+
|
|
64
|
+
def reload(self) -> ConversationEngine:
|
|
65
|
+
"""Rebuild from the files on disk.
|
|
66
|
+
|
|
67
|
+
Raises whatever the build raises (bad YAML, unreadable corpus) *before*
|
|
68
|
+
replacing the running engine, so a failed reload is a no-op.
|
|
69
|
+
"""
|
|
70
|
+
with self._lock:
|
|
71
|
+
rebuilt = _build_engine(self.config)
|
|
72
|
+
# Built before either is assigned: a bad `agent:` section fails the
|
|
73
|
+
# reload like any other bad edit, leaving the running pair alone.
|
|
74
|
+
adapter = build_adapter(self.config, engine=rebuilt)
|
|
75
|
+
self._engine = rebuilt
|
|
76
|
+
self._adapter = adapter
|
|
77
|
+
self.reload_count += 1
|
|
78
|
+
log.info("Engine reloaded (%d chunks indexed)", len(rebuilt.retriever))
|
|
79
|
+
return rebuilt
|
|
80
|
+
|
|
81
|
+
def describe(self) -> dict[str, Any]:
|
|
82
|
+
return {**self._engine.describe(), "reload_count": self.reload_count}
|