askback 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. askback/__init__.py +29 -0
  2. askback/__main__.py +8 -0
  3. askback/adapters/__init__.py +11 -0
  4. askback/adapters/base.py +189 -0
  5. askback/adapters/factory.py +176 -0
  6. askback/adapters/http.py +443 -0
  7. askback/adapters/in_process.py +135 -0
  8. askback/adapters/payload.py +207 -0
  9. askback/adapters/scripted.py +117 -0
  10. askback/adapters/socketio_adapter.py +885 -0
  11. askback/api/__init__.py +18 -0
  12. askback/api/corpus_store.py +85 -0
  13. askback/api/dashboard.py +444 -0
  14. askback/api/server.py +200 -0
  15. askback/api/state.py +82 -0
  16. askback/api/static/app.css +423 -0
  17. askback/api/static/app.js +98 -0
  18. askback/api/static/core.js +201 -0
  19. askback/api/static/index.html +175 -0
  20. askback/api/static/panels.js +651 -0
  21. askback/cli.py +618 -0
  22. askback/core/__init__.py +58 -0
  23. askback/core/config.py +224 -0
  24. askback/core/contracts.py +201 -0
  25. askback/core/errors.py +35 -0
  26. askback/core/logging.py +57 -0
  27. askback/core/registry.py +73 -0
  28. askback/core/text.py +151 -0
  29. askback/datasets/__init__.py +14 -0
  30. askback/datasets/loader.py +268 -0
  31. askback/datasets/schema.py +525 -0
  32. askback/demo/__init__.py +42 -0
  33. askback/demo/agent.py +195 -0
  34. askback/demo/suites/demo.yaml +100 -0
  35. askback/evaluation/__init__.py +70 -0
  36. askback/evaluation/base.py +214 -0
  37. askback/evaluation/calibration.py +405 -0
  38. askback/evaluation/citation_eval.py +580 -0
  39. askback/evaluation/clarification.py +477 -0
  40. askback/evaluation/engine.py +193 -0
  41. askback/evaluation/escalation_eval.py +109 -0
  42. askback/evaluation/gates.py +351 -0
  43. askback/evaluation/judge_cache.py +94 -0
  44. askback/evaluation/llm_judge.py +463 -0
  45. askback/evaluation/patterns.py +48 -0
  46. askback/evaluation/pii_eval.py +252 -0
  47. askback/evaluation/rag_eval.py +168 -0
  48. askback/evaluation/regression.py +438 -0
  49. askback/evaluation/results.py +588 -0
  50. askback/evaluation/rule_based.py +198 -0
  51. askback/evaluation/safety_eval.py +170 -0
  52. askback/providers/__init__.py +10 -0
  53. askback/providers/base.py +157 -0
  54. askback/providers/openai_compat.py +340 -0
  55. askback/py.typed +0 -0
  56. askback/reporting/__init__.py +77 -0
  57. askback/reporting/excel_report.py +117 -0
  58. askback/reporting/html_report.py +283 -0
  59. askback/reporting/json_report.py +27 -0
  60. askback/reporting/markdown_report.py +95 -0
  61. askback/reporting/redaction.py +185 -0
  62. askback/runner/__init__.py +5 -0
  63. askback/runner/responders.py +279 -0
  64. askback/runner/test_runner.py +424 -0
  65. askback-0.1.0.dist-info/METADATA +504 -0
  66. askback-0.1.0.dist-info/RECORD +71 -0
  67. askback-0.1.0.dist-info/WHEEL +5 -0
  68. askback-0.1.0.dist-info/entry_points.txt +2 -0
  69. askback-0.1.0.dist-info/licenses/LICENSE +176 -0
  70. askback-0.1.0.dist-info/licenses/NOTICE +15 -0
  71. askback-0.1.0.dist-info/top_level.txt +1 -0
askback/__init__.py ADDED
@@ -0,0 +1,29 @@
1
+ """An evaluation framework for conversational agents.
2
+
3
+ Drives an agent through test conversations, grades the replies with layered
4
+ evaluators, and gates on the result. The agent is reached through an adapter
5
+ (`askback.adapters`), so the same suite grades a deployed service over HTTP or
6
+ the `askback_sim` fixture in this process. This package never imports `askback_sim`.
7
+
8
+ Extension points:
9
+ Adapter askback.adapters.base.AgentAdapter
10
+ Evaluator askback.evaluation.base.Evaluator
11
+ LLM provider askback.providers.base.LLMProvider
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from .core.config import AppConfig
17
+ from .core.contracts import ChatRequest, ChatResponse, ResponseStatus
18
+ from .core.logging import setup_logging
19
+
20
+ __version__ = "0.1.0"
21
+
22
+ __all__ = [
23
+ "AppConfig",
24
+ "ChatRequest",
25
+ "ChatResponse",
26
+ "ResponseStatus",
27
+ "__version__",
28
+ "setup_logging",
29
+ ]
askback/__main__.py ADDED
@@ -0,0 +1,8 @@
1
+ """Allows `python -m askback ...` as well as the installed `askback` command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .cli import main
6
+
7
+ if __name__ == "__main__":
8
+ raise SystemExit(main())
@@ -0,0 +1,11 @@
1
+ """Adapters: the seam between the harness and the system under test.
2
+
3
+ The runner depends on `AgentAdapter` and nothing more specific, which is what
4
+ lets one suite file grade a simulator and a deployed agent without changing.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from .base import AgentAdapter
10
+
11
+ __all__ = ["AgentAdapter"]
@@ -0,0 +1,189 @@
1
+ """The adapter seam: how the harness reaches the system under test.
2
+
3
+ Everything upstream of this - suites, the runner, evaluators, gates - should be
4
+ indifferent to whether the agent is a simulator in this process or a deployed
5
+ service behind HTTP. An adapter is the one place that difference lives: it takes
6
+ a user turn and hands back a `ChatResponse`, the vocabulary the evaluators
7
+ already grade.
8
+
9
+ `AgentAdapter` is a Protocol, not a base class. An adapter implements a shape; it
10
+ does not inherit from the framework, so a team can write one around their own
11
+ client without subclassing anything here.
12
+
13
+ To add an adapter:
14
+
15
+ class MyAdapter:
16
+ def __enter__(self) -> MyAdapter:
17
+ self._client = open_client() # once per run, not per call
18
+ return self
19
+
20
+ def __exit__(self, *exc: object) -> None:
21
+ self._client.close()
22
+
23
+ def send(self, message, *, session_id, principal="default") -> ChatResponse:
24
+ payload = self._client.ask(message, session=session_id)
25
+ return ChatResponse(reply=payload["text"], conversation_id=session_id)
26
+
27
+ def describe(self) -> dict[str, str]:
28
+ return {"adapter": "my-adapter", "model": self._model}
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import threading
34
+ from types import TracebackType
35
+ from typing import Protocol, TypeVar, runtime_checkable
36
+
37
+ from ..core.contracts import ChatResponse
38
+
39
+ _T = TypeVar("_T")
40
+
41
+
42
+ @runtime_checkable
43
+ class AgentAdapter(Protocol):
44
+ """One system under test, as the runner sees it.
45
+
46
+ Synchronous on purpose. The runner is a thread pool and the work is I/O
47
+ bound, so threads already give the concurrency; an async seam would push an
48
+ event loop into every caller and every adapter author for no measured gain.
49
+
50
+ `isinstance(x, AgentAdapter)` only confirms the four methods exist. It says
51
+ nothing about their signatures - that is what the type checker is for.
52
+ """
53
+
54
+ def __enter__(self) -> AgentAdapter:
55
+ """Open whatever the adapter needs for a whole run, and return it.
56
+
57
+ The lifecycle belongs to the adapter so that an HTTP implementation can
58
+ hold one pooled client for every case instead of paying a connection per
59
+ call. `send` is only valid between `__enter__` and `__exit__`.
60
+ """
61
+ ...
62
+
63
+ def __exit__(
64
+ self,
65
+ exc_type: type[BaseException] | None,
66
+ exc: BaseException | None,
67
+ traceback: TracebackType | None,
68
+ /,
69
+ ) -> None:
70
+ """Release the transport. Runs whether the run finished or blew up.
71
+
72
+ Returns None rather than bool: an adapter has no business suppressing an
73
+ exception raised by the run it was carrying.
74
+ """
75
+ ...
76
+
77
+ def send(
78
+ self, message: str, *, session_id: str, principal: str = "default"
79
+ ) -> ChatResponse:
80
+ """Deliver one user turn and return what the system said.
81
+
82
+ `session_id` is chosen by the caller. Turns sharing one belong to the
83
+ same conversation and must see each other's history; turns with
84
+ different ids must not, which is what keeps test cases from
85
+ contaminating one another.
86
+
87
+ `principal` names the identity the turn is sent as. The adapter resolves
88
+ the name to whatever the system needs - a token, headers, a user and
89
+ tenant. It is a label, never the credential itself: it is recorded in
90
+ reports.
91
+
92
+ MUST be safe to call concurrently from multiple threads. The runner
93
+ sends different sessions from different workers at the same time, so
94
+ anything shared across calls - a client, a token cache, a session map -
95
+ has to tolerate that. Turns within one session arrive in order, never
96
+ interleaved. Nothing in this module can enforce this; it is a contract
97
+ on whoever implements it, and breaking it shows up as cross-case
98
+ contamination rather than as an exception.
99
+
100
+ Report only what the system returned. A field it does not expose stays
101
+ empty - no retrieved chunks, no citations - so the evaluators that need
102
+ it call the case unmeasurable instead of grading an invention. Raise
103
+ when the turn could not be delivered or its reply could not be read; the
104
+ runner records that as a failed case. Do not synthesise a response to
105
+ stand in for one.
106
+ """
107
+ ...
108
+
109
+ def describe(self) -> dict[str, str]:
110
+ """Provenance for the run report: which system this was.
111
+
112
+ The runner cannot reach inside an adapter for a model or retriever name
113
+ the way it could with an in-process engine, so the adapter states its
114
+ own. The values land in `RunReport.context`, are written into every
115
+ report, and are what the regression comparison reads to decide whether
116
+ two runs are comparable at all.
117
+
118
+ The keys are the implementer's choice. Include whatever would make two
119
+ runs differ for reasons other than the agent's behaviour - endpoint,
120
+ model, deployed version, corpus revision - and keep the keys stable from
121
+ run to run: a renamed key compares as nothing. Strings only, because the
122
+ mapping is serialised and compared by equality. Never a secret; reports
123
+ are artifacts.
124
+
125
+ Must not depend on a turn having been sent: the runner records it when
126
+ the run starts.
127
+ """
128
+ ...
129
+
130
+
131
+ class AdapterLifecycle:
132
+ """Reentrant `with` support shared by adapters, so each does not re-derive it.
133
+
134
+ A depth, not a flag: one adapter can be entered by several runs, and the
135
+ first to finish must not close it under the others. A subclass calls
136
+ `super().__init__()`, then guards `send` and `describe` with
137
+ `_require_open`.
138
+
139
+ An adapter that owns a transport (an HTTP client) overrides `_open` and
140
+ `_close`. They run under the counter's lock on the first entry and the
141
+ last exit, so the transport is built once per run, never per `with` or per
142
+ call, and a racing second entry cannot see it half-built. If `_open`
143
+ raises, the adapter stays closed.
144
+ """
145
+
146
+ def __init__(self) -> None:
147
+ self._depth = 0
148
+ # Guards the counter only. `send` is never serialised behind it.
149
+ self._depth_lock = threading.Lock()
150
+
151
+ def __enter__(self: _T) -> _T:
152
+ assert isinstance(self, AdapterLifecycle)
153
+ with self._depth_lock:
154
+ if self._depth == 0:
155
+ self._open()
156
+ self._depth += 1
157
+ return self
158
+
159
+ def __exit__(
160
+ self,
161
+ exc_type: type[BaseException] | None,
162
+ exc: BaseException | None,
163
+ traceback: TracebackType | None,
164
+ /,
165
+ ) -> None:
166
+ with self._depth_lock:
167
+ # Floored so a stray manual __exit__ cannot leave the next `with`
168
+ # block closed from the inside.
169
+ if self._depth == 0:
170
+ return
171
+ self._depth -= 1
172
+ if self._depth == 0:
173
+ self._close()
174
+
175
+ def _open(self) -> None:
176
+ """Acquire the transport. Default: there is none."""
177
+
178
+ def _close(self) -> None:
179
+ """Release the transport. Must tolerate being called after a failed run."""
180
+
181
+ def _require_open(self, method: str) -> None:
182
+ if self._depth == 0:
183
+ raise RuntimeError(
184
+ f"{type(self).__name__}.{method}() called outside its `with` block; "
185
+ "enter the adapter before using it."
186
+ )
187
+
188
+
189
+ __all__ = ["AdapterLifecycle", "AgentAdapter"]
@@ -0,0 +1,176 @@
1
+ """Choosing the adapter from config.
2
+
3
+ `agent.adapter` in your config file names the system under test, so pointing a run
4
+ at something else is a config edit, not a code change. Callers get an
5
+ `AgentAdapter` back and never learn which one it is.
6
+
7
+ This module never imports the simulator, not even lazily. `in_process` needs an
8
+ engine, and the caller - the CLI or the API, which are allowed to know the
9
+ simulator exists - builds it and passes it in. A factory that reached for the
10
+ simulator itself would be the one exception the package boundary could not
11
+ survive having.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from collections.abc import Mapping
17
+ from typing import Any
18
+
19
+ from ..core.config import AppConfig
20
+ from ..core.errors import ConfigError
21
+ from ..evaluation.safety_eval import declare_side_effecting_tools
22
+ from .base import AgentAdapter
23
+ from .in_process import DEFAULT_PRINCIPALS, InProcessAdapter, InProcessEngine
24
+
25
+ #: What a config file with no `agent:` section runs against. Configs written before
26
+ #: the adapter seam existed have none, and they always meant the simulator. A run with no
27
+ #: config file at all never gets here: it is the built-in demo (see `askback.demo`).
28
+ DEFAULT_ADAPTER = "in_process"
29
+
30
+ VALID_ADAPTERS = ("in_process", "http", "socketio", "demo")
31
+
32
+
33
+ def adapter_name(settings: AppConfig) -> str:
34
+ """Which adapter `agent.adapter` names, so a caller knows whether to build an engine."""
35
+ return str(_agent_section(settings).get("adapter", DEFAULT_ADAPTER))
36
+
37
+
38
+ def build_adapter(
39
+ settings: AppConfig, *, engine: InProcessEngine | None = None
40
+ ) -> AgentAdapter:
41
+ """Build the adapter named by `agent.adapter`.
42
+
43
+ `engine` is the in-process system under test, and is required when the
44
+ adapter is `in_process`: this factory does not build one, because that
45
+ would mean importing the simulator. It is ignored by every other adapter.
46
+
47
+ `agent.side_effecting_tools`, when present, is registered here as a
48
+ declaration source for the tool-misuse check. Here rather than in each
49
+ caller because this is the one place every run names its system under
50
+ test, and that system is the only thing that can know its tools.
51
+ """
52
+ agent = _agent_section(settings)
53
+
54
+ tools = _side_effecting_tools(agent)
55
+ if tools is not None:
56
+ declare_side_effecting_tools(lambda: tools)
57
+
58
+ name = agent.get("adapter", DEFAULT_ADAPTER)
59
+ if name == "in_process":
60
+ if engine is None:
61
+ raise ConfigError(
62
+ "agent.adapter 'in_process' drives an engine in this process, and none "
63
+ "was passed to build_adapter(engine=...). The example simulator provides one "
64
+ "from a source checkout: see examples/hr."
65
+ )
66
+ return InProcessAdapter(engine, _principals(agent))
67
+
68
+ if name == "http":
69
+ try:
70
+ from .http import HTTPAdapter
71
+ except ImportError as exc:
72
+ raise ConfigError(
73
+ "agent.adapter 'http' needs httpx. Run: pip install -e .[http]"
74
+ ) from exc
75
+
76
+ return HTTPAdapter.from_config(agent)
77
+
78
+ if name == "demo":
79
+ from ..demo import DemoAdapter
80
+
81
+ return DemoAdapter.from_config(agent)
82
+
83
+ if name == "socketio":
84
+ # No ImportError guard here: that module imports python-socketio when it
85
+ # builds, and turns its absence into a ConfigError carrying the install hint.
86
+ from .socketio_adapter import SocketIOAdapter
87
+
88
+ return SocketIOAdapter.from_config(agent)
89
+
90
+ raise ConfigError(
91
+ f"unknown agent.adapter {name!r}; valid options: {', '.join(VALID_ADAPTERS)}"
92
+ )
93
+
94
+
95
+ def resolve_principals(settings: AppConfig) -> dict[str, tuple[str, str]]:
96
+ """The principal mapping a run will use, for checking datasets before it starts.
97
+
98
+ The configured one, or the adapter's defaults when `agent.principals` is
99
+ absent. Only the in-process adapter resolves identities to tenants; any
100
+ other adapter yields an empty mapping, which makes the loader check a no-op.
101
+ """
102
+ agent = _agent_section(settings)
103
+ if agent.get("adapter", DEFAULT_ADAPTER) != "in_process":
104
+ # An HTTP identity is a header, not a (user_id, tenant_id) pair, so
105
+ # there is no tenant to check a dataset's declaration against.
106
+ return {}
107
+ configured = _principals(agent)
108
+ if configured is not None:
109
+ return configured
110
+ return dict(DEFAULT_PRINCIPALS)
111
+
112
+
113
+ def _agent_section(settings: AppConfig) -> Mapping[str, Any]:
114
+ agent = settings.raw.get("agent", {})
115
+ if not isinstance(agent, Mapping):
116
+ raise ConfigError(f"config `agent` must be a mapping, got {agent!r}")
117
+ return agent
118
+
119
+
120
+ def _principals(agent: Mapping[str, Any]) -> dict[str, tuple[str, str]] | None:
121
+ """Read `agent.principals` into the (user_id, tenant_id) pairs an adapter takes.
122
+
123
+ None when the key is absent, which leaves the adapter on its own defaults.
124
+ Anything present is checked here so a malformed entry is reported with its
125
+ config path, before a case is sent as it.
126
+ """
127
+ declared = agent.get("principals")
128
+ if declared is None:
129
+ return None
130
+ if not isinstance(declared, Mapping):
131
+ raise ConfigError(
132
+ "agent.principals must map a principal label to "
133
+ f"{{user_id, tenant_id}}, got {declared!r}"
134
+ )
135
+
136
+ principals: dict[str, tuple[str, str]] = {}
137
+ for label, identity in declared.items():
138
+ if not (
139
+ isinstance(label, str)
140
+ and isinstance(identity, Mapping)
141
+ and set(identity) == {"user_id", "tenant_id"}
142
+ and all(isinstance(part, str) for part in identity.values())
143
+ ):
144
+ raise ConfigError(
145
+ f"agent.principals.{label} must be a mapping with exactly a string "
146
+ f"user_id and a string tenant_id, got {identity!r}"
147
+ )
148
+ principals[label] = (identity["user_id"], identity["tenant_id"])
149
+ return principals
150
+
151
+
152
+ def _side_effecting_tools(agent: Mapping[str, Any]) -> frozenset[str] | None:
153
+ """Read `agent.side_effecting_tools`: the tools this agent says act on the world.
154
+
155
+ None when the key is absent, and absent is not the same as empty: `[]`
156
+ states that no tool has side effects, and the check can then pass. Absent
157
+ states nothing, so the check keeps failing until someone does (invariant 8).
158
+ A bare string is refused; iterating it would declare its letters.
159
+ """
160
+ declared = agent.get("side_effecting_tools")
161
+ if declared is None:
162
+ return None
163
+ if not isinstance(declared, list) or not all(isinstance(name, str) for name in declared):
164
+ raise ConfigError(
165
+ f"agent.side_effecting_tools must be a list of tool names, got {declared!r}"
166
+ )
167
+ return frozenset(declared)
168
+
169
+
170
+ __all__ = [
171
+ "DEFAULT_ADAPTER",
172
+ "VALID_ADAPTERS",
173
+ "adapter_name",
174
+ "build_adapter",
175
+ "resolve_principals",
176
+ ]