askback 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. askback/__init__.py +29 -0
  2. askback/__main__.py +8 -0
  3. askback/adapters/__init__.py +11 -0
  4. askback/adapters/base.py +189 -0
  5. askback/adapters/factory.py +176 -0
  6. askback/adapters/http.py +443 -0
  7. askback/adapters/in_process.py +135 -0
  8. askback/adapters/payload.py +207 -0
  9. askback/adapters/scripted.py +117 -0
  10. askback/adapters/socketio_adapter.py +885 -0
  11. askback/api/__init__.py +18 -0
  12. askback/api/corpus_store.py +85 -0
  13. askback/api/dashboard.py +444 -0
  14. askback/api/server.py +200 -0
  15. askback/api/state.py +82 -0
  16. askback/api/static/app.css +423 -0
  17. askback/api/static/app.js +98 -0
  18. askback/api/static/core.js +201 -0
  19. askback/api/static/index.html +175 -0
  20. askback/api/static/panels.js +651 -0
  21. askback/cli.py +618 -0
  22. askback/core/__init__.py +58 -0
  23. askback/core/config.py +224 -0
  24. askback/core/contracts.py +201 -0
  25. askback/core/errors.py +35 -0
  26. askback/core/logging.py +57 -0
  27. askback/core/registry.py +73 -0
  28. askback/core/text.py +151 -0
  29. askback/datasets/__init__.py +14 -0
  30. askback/datasets/loader.py +268 -0
  31. askback/datasets/schema.py +525 -0
  32. askback/demo/__init__.py +42 -0
  33. askback/demo/agent.py +195 -0
  34. askback/demo/suites/demo.yaml +100 -0
  35. askback/evaluation/__init__.py +70 -0
  36. askback/evaluation/base.py +214 -0
  37. askback/evaluation/calibration.py +405 -0
  38. askback/evaluation/citation_eval.py +580 -0
  39. askback/evaluation/clarification.py +477 -0
  40. askback/evaluation/engine.py +193 -0
  41. askback/evaluation/escalation_eval.py +109 -0
  42. askback/evaluation/gates.py +351 -0
  43. askback/evaluation/judge_cache.py +94 -0
  44. askback/evaluation/llm_judge.py +463 -0
  45. askback/evaluation/patterns.py +48 -0
  46. askback/evaluation/pii_eval.py +252 -0
  47. askback/evaluation/rag_eval.py +168 -0
  48. askback/evaluation/regression.py +438 -0
  49. askback/evaluation/results.py +588 -0
  50. askback/evaluation/rule_based.py +198 -0
  51. askback/evaluation/safety_eval.py +170 -0
  52. askback/providers/__init__.py +10 -0
  53. askback/providers/base.py +157 -0
  54. askback/providers/openai_compat.py +340 -0
  55. askback/py.typed +0 -0
  56. askback/reporting/__init__.py +77 -0
  57. askback/reporting/excel_report.py +117 -0
  58. askback/reporting/html_report.py +283 -0
  59. askback/reporting/json_report.py +27 -0
  60. askback/reporting/markdown_report.py +95 -0
  61. askback/reporting/redaction.py +185 -0
  62. askback/runner/__init__.py +5 -0
  63. askback/runner/responders.py +279 -0
  64. askback/runner/test_runner.py +424 -0
  65. askback-0.1.0.dist-info/METADATA +504 -0
  66. askback-0.1.0.dist-info/RECORD +71 -0
  67. askback-0.1.0.dist-info/WHEEL +5 -0
  68. askback-0.1.0.dist-info/entry_points.txt +2 -0
  69. askback-0.1.0.dist-info/licenses/LICENSE +176 -0
  70. askback-0.1.0.dist-info/licenses/NOTICE +15 -0
  71. askback-0.1.0.dist-info/top_level.txt +1 -0
askback/api/server.py ADDED
@@ -0,0 +1,200 @@
1
+ """FastAPI application: the public API, plus the built-in dashboard.
2
+
3
+ The engine is built once at startup and reused: loading the corpus per request
4
+ would dominate latency. `InMemoryConversationStore` is per process, so run a
5
+ single worker locally, and swap in a shared `ConversationStore` before scaling
6
+ out (see `askback_sim.memory`).
7
+
8
+ askback serve # dashboard at /, API docs at /docs
9
+ curl -X POST localhost:8000/chat -H "content-type: application/json" \
10
+ -d '{"message": "How many days do I have to return an item?"}'
11
+
12
+ Two surfaces, deliberately separate:
13
+
14
+ /chat, /info, /search the stable contract other systems integrate against
15
+ /api/* developer tooling behind the dashboard, free to change
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ from fastapi import Depends, FastAPI, Header, HTTPException, status
24
+ from fastapi.middleware.cors import CORSMiddleware
25
+ from fastapi.responses import FileResponse
26
+ from fastapi.staticfiles import StaticFiles
27
+ from pydantic import BaseModel, Field
28
+
29
+ from .. import __version__
30
+ from ..core.config import AppConfig, resolve_path
31
+ from ..core.contracts import ChatRequest
32
+ from ..core.logging import get_logger, setup_logging
33
+ from .dashboard import build_dashboard_router, safe_name
34
+ from .state import EngineHolder
35
+
36
+ log = get_logger(__name__)
37
+
38
+ STATIC_DIR = Path(__file__).parent / "static"
39
+
40
+
41
+ # --------------------------------------------------------------------------- #
42
+ # Wire models. Separate from the internal contracts on purpose: the HTTP surface
43
+ # should be free to stay stable while the internals evolve.
44
+ # --------------------------------------------------------------------------- #
45
+
46
+
47
+ class ChatIn(BaseModel):
48
+ message: str = Field(..., min_length=1, max_length=4000)
49
+ conversation_id: str | None = Field(None, max_length=128)
50
+ user_id: str = Field("api_user", max_length=128)
51
+ tenant_id: str = Field("default", max_length=128)
52
+ metadata: dict[str, Any] = Field(default_factory=dict)
53
+
54
+
55
+ class CitationOut(BaseModel):
56
+ doc_id: str
57
+ title: str = ""
58
+ source: str = ""
59
+ snippet: str = ""
60
+
61
+
62
+ class ChatOut(BaseModel):
63
+ reply: str
64
+ conversation_id: str
65
+ route: str
66
+ status: str
67
+ intent: str
68
+ intent_confidence: float
69
+ citations: list[CitationOut] = Field(default_factory=list)
70
+ tools_used: list[str] = Field(default_factory=list)
71
+ safety_blocked: bool = False
72
+ latency_ms: float = 0.0
73
+ trace_id: str = ""
74
+
75
+
76
+ class ResetIn(BaseModel):
77
+ conversation_id: str | None = None
78
+
79
+
80
+ def create_app(config: AppConfig | None = None) -> FastAPI:
81
+ """Build the application. Import-safe: nothing runs until this is called."""
82
+ cfg = config or AppConfig.load()
83
+ setup_logging(str(cfg.logging.get("level", "INFO")))
84
+
85
+ app = FastAPI(
86
+ title="askback dashboard",
87
+ version=__version__,
88
+ description="Configurable conversational agent with routing, RAG, safety and evaluation.",
89
+ )
90
+
91
+ origins = cfg.server.get("cors_origins", ["*"])
92
+ app.add_middleware(
93
+ CORSMiddleware,
94
+ allow_origins=origins,
95
+ allow_methods=["GET", "POST"],
96
+ allow_headers=["*"],
97
+ )
98
+
99
+ # Built once, shared by every request. The holder exists so the dashboard can
100
+ # rebuild it after the files change on disk, without restarting the server.
101
+ holder = EngineHolder(cfg)
102
+ expected_key = str(cfg.server.get("api_key", "") or "")
103
+
104
+ def require_api_key(x_api_key: str | None = Header(default=None)) -> None:
105
+ """No-op when `server.api_key` is unset, which is the local default."""
106
+ if expected_key and x_api_key != expected_key:
107
+ raise HTTPException(
108
+ status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or missing API key."
109
+ )
110
+
111
+ @app.get("/health", tags=["ops"])
112
+ def health() -> dict[str, Any]:
113
+ """Liveness probe. Cheap enough for a load balancer to hit often."""
114
+ return {"status": "ok", "version": app.version}
115
+
116
+ @app.get("/info", tags=["ops"], dependencies=[Depends(require_api_key)])
117
+ def info() -> dict[str, Any]:
118
+ """How the engine is wired: provider, retriever, agents, tools, guards."""
119
+ return holder.engine.describe()
120
+
121
+ @app.post("/chat", response_model=ChatOut, tags=["chat"], dependencies=[Depends(require_api_key)])
122
+ def chat(payload: ChatIn) -> ChatOut:
123
+ """Answer one message."""
124
+ request = ChatRequest(
125
+ message=payload.message,
126
+ user_id=payload.user_id,
127
+ tenant_id=payload.tenant_id,
128
+ metadata=payload.metadata,
129
+ **({"conversation_id": payload.conversation_id} if payload.conversation_id else {}),
130
+ )
131
+ response = holder.engine.chat(request)
132
+ return ChatOut(
133
+ reply=response.reply,
134
+ conversation_id=response.conversation_id,
135
+ route=response.route,
136
+ status=response.status.value,
137
+ intent=response.intent.name,
138
+ intent_confidence=round(response.intent.confidence, 4),
139
+ citations=[
140
+ CitationOut(
141
+ doc_id=c.doc_id, title=c.title, source=c.source, snippet=c.snippet[:240]
142
+ )
143
+ for c in response.citations
144
+ ],
145
+ tools_used=response.tool_names,
146
+ safety_blocked=response.safety.blocked,
147
+ latency_ms=response.latency_ms,
148
+ trace_id=response.trace_id,
149
+ )
150
+
151
+ @app.post("/reset", tags=["chat"], dependencies=[Depends(require_api_key)])
152
+ def reset(payload: ResetIn) -> dict[str, str]:
153
+ """Clear one conversation, or every conversation when no id is given."""
154
+ holder.engine.reset(payload.conversation_id)
155
+ return {"status": "reset", "conversation_id": payload.conversation_id or "all"}
156
+
157
+ @app.get("/search", tags=["debug"], dependencies=[Depends(require_api_key)])
158
+ def search(q: str, top_k: int = 5, tenant: str = "default") -> dict[str, Any]:
159
+ """Inspect retrieval directly. Useful when an answer looks wrong."""
160
+ hits = holder.engine.retriever.search(q, top_k=min(top_k, 20), tenant=tenant)
161
+ return {
162
+ "query": q,
163
+ "hits": [
164
+ {"doc_id": h.doc_id, "title": h.title, "score": h.score, "text": h.text[:300]}
165
+ for h in hits
166
+ ],
167
+ }
168
+
169
+ app.include_router(build_dashboard_router(holder, cfg, require_api_key))
170
+ _mount_dashboard(app, cfg)
171
+ return app
172
+
173
+
174
+ def _mount_dashboard(app: FastAPI, cfg: AppConfig) -> None:
175
+ """Serve the dashboard at `/`, if its assets shipped with the package.
176
+
177
+ A slim install without the static files should still serve the API rather
178
+ than fail at startup, so a missing directory is a warning, not an error.
179
+ """
180
+ if not STATIC_DIR.is_dir():
181
+ log.warning("Dashboard assets not found at %s; serving API only.", STATIC_DIR)
182
+ return
183
+
184
+ app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
185
+
186
+ @app.get("/", include_in_schema=False)
187
+ def dashboard() -> FileResponse:
188
+ return FileResponse(STATIC_DIR / "index.html")
189
+
190
+ @app.get("/reports/{name}.html", include_in_schema=False)
191
+ def report_page(name: str) -> FileResponse:
192
+ """Open a generated HTML report from the dashboard's run list."""
193
+ path = resolve_path(cfg.evaluation.get("reports_dir", "reports")) / f"{safe_name(name)}.html"
194
+ if not path.exists():
195
+ raise HTTPException(status.HTTP_404_NOT_FOUND, f"No HTML report named {name!r}.")
196
+ return FileResponse(path)
197
+
198
+
199
+ # Uvicorn entry point: `uvicorn askback.api.server:app`
200
+ app = None # set lazily by `create_app`; `askback serve` builds it explicitly
askback/api/state.py ADDED
@@ -0,0 +1,82 @@
1
+ """Holds the live engine so the dashboard can rebuild it without a restart.
2
+
3
+ The engine is expensive to build (it indexes the whole corpus), so the service
4
+ builds one at startup and shares it. The corpus and the intent catalogue are
5
+ edited as files on disk, which means something has to swap the engine
6
+ afterwards; that is this.
7
+
8
+ Swapping is atomic from a request's point of view: a new engine is built first
9
+ and only assigned once it succeeds, so a broken edit leaves the running engine
10
+ untouched and the error goes back to whoever asked for the reload.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import threading
16
+ from typing import TYPE_CHECKING, Any
17
+
18
+ from ..adapters import AgentAdapter
19
+ from ..adapters.factory import build_adapter
20
+ from ..core.config import AppConfig
21
+ from ..core.logging import get_logger
22
+
23
+ if TYPE_CHECKING:
24
+ from askback_sim.engine import ConversationEngine
25
+
26
+ log = get_logger(__name__)
27
+
28
+
29
+ def _build_engine(config: AppConfig) -> ConversationEngine:
30
+ """Build the simulator's engine, importing it only when one is built.
31
+
32
+ The adapter factory will not import the simulator, so this is where the
33
+ server - which is allowed to know it exists - does, and hands the engine on.
34
+ """
35
+ from askback_sim.engine import ConversationEngine
36
+
37
+ return ConversationEngine.from_config(config)
38
+
39
+
40
+ class EngineHolder:
41
+ """One mutable reference to the current engine."""
42
+
43
+ def __init__(self, config: AppConfig) -> None:
44
+ self.config = config
45
+ self._lock = threading.Lock()
46
+ self._engine = _build_engine(config)
47
+ self._adapter = build_adapter(config, engine=self._engine)
48
+ self.reload_count = 0
49
+
50
+ @property
51
+ def engine(self) -> ConversationEngine:
52
+ """The engine to serve this request with."""
53
+ return self._engine
54
+
55
+ @property
56
+ def adapter(self) -> AgentAdapter:
57
+ """The current engine as the evaluation runner reaches it.
58
+
59
+ Shared by every run: adapters count their entries, so concurrent runs
60
+ can each hold it open.
61
+ """
62
+ return self._adapter
63
+
64
+ def reload(self) -> ConversationEngine:
65
+ """Rebuild from the files on disk.
66
+
67
+ Raises whatever the build raises (bad YAML, unreadable corpus) *before*
68
+ replacing the running engine, so a failed reload is a no-op.
69
+ """
70
+ with self._lock:
71
+ rebuilt = _build_engine(self.config)
72
+ # Built before either is assigned: a bad `agent:` section fails the
73
+ # reload like any other bad edit, leaving the running pair alone.
74
+ adapter = build_adapter(self.config, engine=rebuilt)
75
+ self._engine = rebuilt
76
+ self._adapter = adapter
77
+ self.reload_count += 1
78
+ log.info("Engine reloaded (%d chunks indexed)", len(rebuilt.retriever))
79
+ return rebuilt
80
+
81
+ def describe(self) -> dict[str, Any]:
82
+ return {**self._engine.describe(), "reload_count": self.reload_count}