evalloop-agents 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
evalloop/__init__.py ADDED
@@ -0,0 +1,45 @@
1
+ """evalloop: unit tests and a CI gate for LLM agents."""
2
+ from .checks import (
3
+ Check,
4
+ CheckResult,
5
+ answer_contains,
6
+ answer_matches,
7
+ answer_not_contains,
8
+ cost_under,
9
+ custom,
10
+ grounded,
11
+ json_answer,
12
+ judge,
13
+ latency_under,
14
+ max_steps,
15
+ no_loop,
16
+ no_pii,
17
+ no_secret_leak,
18
+ no_tool_errors,
19
+ tool_args,
20
+ tool_called,
21
+ tool_count,
22
+ tool_not_called,
23
+ tool_order,
24
+ )
25
+ from .adapters import AdapterError, anthropic_agent, chat_agent, from_provider, judge_with
26
+ from .core import Step, Trace, Tracer
27
+ from .report import render_html
28
+ from .traces import from_anthropic_messages, from_openai_messages, load_traces, replay_agent, save_traces
29
+ from .ci import assert_reliable, to_junit, to_markdown
30
+ from .redteam import INJECTIONS, injection_cases, perturb, variants
31
+ from .instrument import agent, current, log_llm, poisoning, record, tool
32
+ from .instrument import apply_poison as poisoned
33
+ from .runner import Case, Context, Conversation, Turn, SuiteResult, diff_runs, run_suite
34
+
35
+ __version__ = "0.6.0"
36
+
37
+ __all__ = [
38
+ "AdapterError", "poisoned", "INJECTIONS", "injection_cases", "perturb", "variants", "agent", "tool", "log_llm", "record", "current", "poisoning", "Case", "Conversation", "Turn", "Check", "CheckResult", "Context", "Step", "SuiteResult", "Trace", "Tracer",
39
+ "answer_contains", "answer_matches", "answer_not_contains", "cost_under", "custom",
40
+ "chat_agent", "diff_runs", "from_provider", "grounded", "json_answer", "judge", "latency_under", "max_steps", "no_loop",
41
+ "no_pii", "no_secret_leak", "no_tool_errors", "render_html", "run_suite", "tool_args",
42
+ "tool_called", "tool_count", "tool_not_called", "tool_order",
43
+ "anthropic_agent", "judge_with", "from_openai_messages", "from_anthropic_messages",
44
+ "load_traces", "save_traces", "replay_agent", "to_junit", "to_markdown", "assert_reliable",
45
+ ]
evalloop/adapters.py ADDED
@@ -0,0 +1,287 @@
1
+ """Run a real model through evalloop without extra dependencies.
2
+
3
+ `chat_agent` builds an agent function for any OpenAI-compatible chat endpoint
4
+ (OpenAI, Groq, OpenRouter, Gemini, Ollama, vLLM and others). It runs the usual
5
+ tool-calling loop and records every LLM call and tool call with a Tracer, so all
6
+ the checks work unchanged.
7
+
8
+ The API key is read from an environment variable and is only ever sent in the
9
+ Authorization header. It is never stored, printed or included in error messages.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import os
15
+ import time
16
+ import urllib.error
17
+ import urllib.request
18
+ from typing import Any, Callable, Optional
19
+
20
+ from .core import Tracer
21
+
22
+ # name -> (base url, env var holding the key, a model that is currently offered)
23
+ PROVIDERS = {
24
+ "openai": ("https://api.openai.com/v1", "OPENAI_API_KEY", "gpt-4o-mini"),
25
+ "groq": ("https://api.groq.com/openai/v1", "GROQ_API_KEY", "openai/gpt-oss-120b"),
26
+ "openrouter": ("https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", "meta-llama/llama-3.3-70b-instruct:free"),
27
+ "gemini": ("https://generativelanguage.googleapis.com/v1beta/openai", "GEMINI_API_KEY", "gemini-2.0-flash"),
28
+ "ollama": ("http://localhost:11434/v1", "", "llama3.1"),
29
+ }
30
+
31
+ _RETRY_STATUS = {429, 500, 502, 503, 504}
32
+
33
+
34
+ class AdapterError(RuntimeError):
35
+ """Raised when the provider returns an error. Never contains the API key."""
36
+
37
+ def __init__(self, message: str, status: Optional[int] = None):
38
+ super().__init__(message)
39
+ self.status = status
40
+
41
+
42
+ def _http_post(url: str, payload: dict, headers: dict, timeout: float) -> dict:
43
+ req = urllib.request.Request(url, data=json.dumps(payload).encode(), headers=headers, method="POST")
44
+ try:
45
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
46
+ return json.loads(resp.read().decode())
47
+ except urllib.error.HTTPError as exc:
48
+ body = exc.read().decode(errors="replace")[:200]
49
+ raise AdapterError(f"HTTP {exc.code}: {body}", exc.code) from None
50
+ except urllib.error.URLError as exc:
51
+ raise AdapterError(f"could not reach {url}: {exc.reason}") from None
52
+
53
+
54
+ def chat_agent(
55
+ *,
56
+ system: str,
57
+ tools: list,
58
+ impls: dict,
59
+ model: str,
60
+ base_url: str = "https://api.openai.com/v1",
61
+ api_key: Optional[str] = None,
62
+ api_key_env: str = "OPENAI_API_KEY",
63
+ temperature: float = 0.7,
64
+ max_turns: int = 6,
65
+ price_in: float = 0.0,
66
+ price_out: float = 0.0,
67
+ timeout: float = 60.0,
68
+ retries: int = 2,
69
+ post: Optional[Callable[..., dict]] = None,
70
+ sleep: Callable[[float], None] = time.sleep,
71
+ ) -> Callable:
72
+ """Return `agent(text, ctx)` for run_suite.
73
+
74
+ tools: OpenAI-format tool schemas. impls: tool name -> python function taking the
75
+ tool arguments as keyword arguments. price_in and price_out are USD per 1M tokens
76
+ (leave 0 for free tiers, so cost checks stay meaningful only when you set them).
77
+ """
78
+ key = api_key if api_key is not None else (os.environ.get(api_key_env, "") if api_key_env else "")
79
+ send = post or _http_post
80
+ url = base_url.rstrip("/") + "/chat/completions"
81
+
82
+ def call(messages: list) -> dict:
83
+ headers = {"Content-Type": "application/json"}
84
+ if key:
85
+ headers["Authorization"] = "Bearer " + key
86
+ payload = {"model": model, "messages": messages, "tools": tools, "tool_choice": "auto", "temperature": temperature}
87
+ for attempt in range(retries + 1):
88
+ try:
89
+ return send(url, payload, headers, timeout)
90
+ except AdapterError as exc:
91
+ if exc.status in _RETRY_STATUS and attempt < retries:
92
+ sleep(2 ** attempt * 2)
93
+ continue
94
+ raise
95
+ raise AdapterError("unreachable")
96
+
97
+ def agent(text: str, ctx: Any = None):
98
+ t = Tracer(text)
99
+ messages = [{"role": "system", "content": system}]
100
+ for said, answered in getattr(ctx, "history", None) or []:
101
+ messages += [{"role": "user", "content": said}, {"role": "assistant", "content": answered}]
102
+ messages.append({"role": "user", "content": text})
103
+ for _ in range(max_turns):
104
+ start = time.perf_counter()
105
+ resp = call(messages)
106
+ ms = (time.perf_counter() - start) * 1000
107
+ msg = resp["choices"][0]["message"]
108
+ usage = resp.get("usage") or {}
109
+ calls = msg.get("tool_calls") or []
110
+ t.llm(
111
+ "plan" if calls else "answer",
112
+ messages[-1].get("content") or "",
113
+ msg.get("content") or [c["function"]["name"] for c in calls],
114
+ tokens_in=usage.get("prompt_tokens"),
115
+ tokens_out=usage.get("completion_tokens"),
116
+ latency_ms=ms,
117
+ price_in=price_in,
118
+ price_out=price_out,
119
+ )
120
+ if not calls:
121
+ return t.finish(msg.get("content") or "")
122
+ messages.append({"role": "assistant", "content": msg.get("content"), "tool_calls": calls})
123
+ for c in calls:
124
+ name = c["function"]["name"]
125
+ try:
126
+ args = json.loads(c["function"].get("arguments") or "{}")
127
+ except ValueError:
128
+ args = {}
129
+ fn = impls.get(name)
130
+ if fn is None:
131
+ out = t.tool(name, args, output={"error": f"unknown tool {name}"})
132
+ else:
133
+ out = t.tool(name, args, fn=fn)
134
+ if t.trace.steps[-1].error:
135
+ out = {"error": t.trace.steps[-1].error}
136
+ messages.append({"role": "tool", "tool_call_id": c.get("id", name), "content": json.dumps(out, default=str)})
137
+ trace = t.finish("")
138
+ trace.error = f"no final answer after {max_turns} turns"
139
+ return trace
140
+
141
+ return agent
142
+
143
+
144
+ def from_provider(provider: str, *, system: str, tools: list, impls: dict, model: Optional[str] = None, **kw) -> Callable:
145
+ """Shortcut: from_provider("groq", system=..., tools=..., impls=...). The key comes from GROQ_API_KEY."""
146
+ if provider not in PROVIDERS:
147
+ raise ValueError(f"unknown provider {provider!r}; choose one of {', '.join(PROVIDERS)}")
148
+ base, env, default_model = PROVIDERS[provider]
149
+ return chat_agent(system=system, tools=tools, impls=impls, model=model or default_model, base_url=base, api_key_env=env, **kw)
150
+
151
+
152
+ # ---------------------------------------------------------------- Anthropic (native Messages API)
153
+
154
+ ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"
155
+
156
+
157
+ def anthropic_agent(
158
+ *,
159
+ system: str,
160
+ tools: list,
161
+ impls: dict,
162
+ model: str,
163
+ api_key: Optional[str] = None,
164
+ api_key_env: str = "ANTHROPIC_API_KEY",
165
+ max_tokens: int = 1024,
166
+ temperature: float = 0.7,
167
+ max_turns: int = 6,
168
+ price_in: float = 0.0,
169
+ price_out: float = 0.0,
170
+ timeout: float = 60.0,
171
+ retries: int = 2,
172
+ base_url: str = ANTHROPIC_URL,
173
+ post: Optional[Callable[..., dict]] = None,
174
+ sleep: Callable[[float], None] = time.sleep,
175
+ ) -> Callable:
176
+ """Same as chat_agent, but for Claude through the native Messages API.
177
+
178
+ tools: Anthropic-format schemas ({"name", "description", "input_schema"}).
179
+ The key is read from ANTHROPIC_API_KEY and only goes in the x-api-key header.
180
+ """
181
+ key = api_key if api_key is not None else os.environ.get(api_key_env, "")
182
+ send = post or _http_post
183
+
184
+ def call(messages: list) -> dict:
185
+ headers = {"Content-Type": "application/json", "anthropic-version": "2023-06-01"}
186
+ if key:
187
+ headers["x-api-key"] = key
188
+ payload = {
189
+ "model": model,
190
+ "system": system,
191
+ "messages": messages,
192
+ "tools": tools,
193
+ "max_tokens": max_tokens,
194
+ "temperature": temperature,
195
+ }
196
+ for attempt in range(retries + 1):
197
+ try:
198
+ return send(base_url, payload, headers, timeout)
199
+ except AdapterError as exc:
200
+ if (exc.status in _RETRY_STATUS or exc.status == 529) and attempt < retries:
201
+ sleep(2 ** attempt * 2)
202
+ continue
203
+ raise
204
+ raise AdapterError("unreachable")
205
+
206
+ def agent(text: str, ctx: Any = None):
207
+ t = Tracer(text)
208
+ messages: list = []
209
+ for said, answered in getattr(ctx, "history", None) or []:
210
+ messages += [{"role": "user", "content": said}, {"role": "assistant", "content": answered}]
211
+ messages.append({"role": "user", "content": text})
212
+ for _ in range(max_turns):
213
+ start = time.perf_counter()
214
+ resp = call(messages)
215
+ ms = (time.perf_counter() - start) * 1000
216
+ blocks = resp.get("content") or []
217
+ uses = [b for b in blocks if b.get("type") == "tool_use"]
218
+ said = "".join(b.get("text", "") for b in blocks if b.get("type") == "text")
219
+ usage = resp.get("usage") or {}
220
+ t.llm(
221
+ "plan" if uses else "answer",
222
+ messages[-1]["content"] if isinstance(messages[-1]["content"], str) else "tool results",
223
+ said or [u["name"] for u in uses],
224
+ tokens_in=usage.get("input_tokens"),
225
+ tokens_out=usage.get("output_tokens"),
226
+ latency_ms=ms,
227
+ price_in=price_in,
228
+ price_out=price_out,
229
+ )
230
+ if not uses:
231
+ return t.finish(said)
232
+ messages.append({"role": "assistant", "content": blocks})
233
+ results = []
234
+ for u in uses:
235
+ fn = impls.get(u["name"])
236
+ args = u.get("input") or {}
237
+ if fn is None:
238
+ out = t.tool(u["name"], args, output={"error": f"unknown tool {u['name']}"})
239
+ else:
240
+ out = t.tool(u["name"], args, fn=fn)
241
+ if t.trace.steps[-1].error:
242
+ out = {"error": t.trace.steps[-1].error}
243
+ results.append({"type": "tool_result", "tool_use_id": u["id"], "content": json.dumps(out, default=str)})
244
+ messages.append({"role": "user", "content": results})
245
+ trace = t.finish("")
246
+ trace.error = f"no final answer after {max_turns} turns"
247
+ return trace
248
+
249
+ return agent
250
+
251
+
252
+ def judge_with(provider: str, model: Optional[str] = None, *, api_key: Optional[str] = None,
253
+ post: Optional[Callable[..., dict]] = None, timeout: float = 60.0) -> Callable[[str], str]:
254
+ """Build the `ask(prompt) -> text` function that the `judge` check needs.
255
+
256
+ provider is "anthropic" or any name in PROVIDERS (openai, groq, openrouter, gemini, ollama).
257
+ Uses temperature 0 so the same answer gets the same score.
258
+ """
259
+ send = post or _http_post
260
+ if provider == "anthropic":
261
+ key = api_key if api_key is not None else os.environ.get("ANTHROPIC_API_KEY", "")
262
+
263
+ def ask_anthropic(prompt: str) -> str:
264
+ headers = {"Content-Type": "application/json", "anthropic-version": "2023-06-01"}
265
+ if key:
266
+ headers["x-api-key"] = key
267
+ payload = {"model": model or "claude-haiku-4-5", "max_tokens": 300, "temperature": 0,
268
+ "messages": [{"role": "user", "content": prompt}]}
269
+ resp = send(ANTHROPIC_URL, payload, headers, timeout)
270
+ return "".join(b.get("text", "") for b in resp.get("content", []) if b.get("type") == "text")
271
+
272
+ return ask_anthropic
273
+ if provider not in PROVIDERS:
274
+ raise ValueError(f"unknown provider {provider!r}")
275
+ base, env, default_model = PROVIDERS[provider]
276
+ key = api_key if api_key is not None else (os.environ.get(env, "") if env else "")
277
+
278
+ def ask(prompt: str) -> str:
279
+ headers = {"Content-Type": "application/json"}
280
+ if key:
281
+ headers["Authorization"] = "Bearer " + key
282
+ payload = {"model": model or default_model, "temperature": 0,
283
+ "messages": [{"role": "user", "content": prompt}]}
284
+ resp = send(base.rstrip("/") + "/chat/completions", payload, headers, timeout)
285
+ return resp["choices"][0]["message"].get("content") or ""
286
+
287
+ return ask
evalloop/checks.py ADDED
@@ -0,0 +1,320 @@
1
+ """Built-in checks. Each factory returns a Check you can attach to a Case.
2
+
3
+ A check function receives (trace, case) and returns either a bool or a
4
+ (bool, detail) tuple. Raising inside a check counts as a failure.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import dataclasses as dc
9
+ import json
10
+ import re
11
+ from typing import Any, Callable
12
+
13
+ from .core import Trace, text_of
14
+
15
+
16
+ @dc.dataclass
17
+ class CheckResult:
18
+ name: str
19
+ passed: bool
20
+ detail: str = ""
21
+
22
+
23
+ @dc.dataclass(frozen=True)
24
+ class Check:
25
+ name: str
26
+ fn: Callable[[Trace, Any], Any]
27
+
28
+ def __call__(self, trace: Trace, case: Any) -> CheckResult:
29
+ try:
30
+ out = self.fn(trace, case)
31
+ except Exception as exc:
32
+ return CheckResult(self.name, False, f"check raised {type(exc).__name__}: {exc}")
33
+ if isinstance(out, tuple):
34
+ ok, detail = out
35
+ return CheckResult(self.name, bool(ok), str(detail))
36
+ return CheckResult(self.name, bool(out), "")
37
+
38
+
39
+ def custom(name: str, fn: Callable[[Trace, Any], Any]) -> Check:
40
+ """Wrap any function as a check."""
41
+ return Check(name, fn)
42
+
43
+
44
+ # ---------------------------------------------------------------- tool use
45
+
46
+
47
+ def tool_called(name: str, times: int | None = None) -> Check:
48
+ def fn(trace, case):
49
+ n = sum(1 for s in trace.tool_calls if s.name == name)
50
+ ok = n > 0 if times is None else n == times
51
+ return ok, f"{name} called {n}x" + (f", expected {times}x" if times is not None else "")
52
+
53
+ return Check(f"tool_called({name})" + (f" x{times}" if times is not None else ""), fn)
54
+
55
+
56
+ def tool_not_called(name: str) -> Check:
57
+ def fn(trace, case):
58
+ n = sum(1 for s in trace.tool_calls if s.name == name)
59
+ return n == 0, f"{name} called {n}x"
60
+
61
+ return Check(f"tool_not_called({name})", fn)
62
+
63
+
64
+ def tool_args(name: str, **expected: Any) -> Check:
65
+ """Some call to `name` must contain all the expected argument values.
66
+
67
+ A value can be a plain value, a compiled regex, or a predicate function.
68
+ """
69
+
70
+ def fn(trace, case):
71
+ calls = [s for s in trace.tool_calls if s.name == name]
72
+ if not calls:
73
+ return False, f"{name} was never called"
74
+ for s in calls:
75
+ args = s.input if isinstance(s.input, dict) else {}
76
+ if all(k in args and _same(args[k], v) for k, v in expected.items()):
77
+ return True, ""
78
+ seen = [s.input for s in calls]
79
+ return False, f"expected {expected}, saw {seen}"
80
+
81
+ return Check(f"tool_args({name})", fn)
82
+
83
+
84
+ def tool_order(*names: str) -> Check:
85
+ """These tools must appear in this relative order (other calls may sit between)."""
86
+
87
+ def fn(trace, case):
88
+ seq = [s.name for s in trace.tool_calls]
89
+ i = 0
90
+ for n in seq:
91
+ if i < len(names) and n == names[i]:
92
+ i += 1
93
+ return i == len(names), f"saw {seq}"
94
+
95
+ return Check("tool_order(" + " > ".join(names) + ")", fn)
96
+
97
+
98
+ def no_loop(max_repeats: int = 2) -> Check:
99
+ """Fail if the same tool is called with the same args more than `max_repeats` times."""
100
+
101
+ def fn(trace, case):
102
+ counts: dict[str, int] = {}
103
+ for s in trace.tool_calls:
104
+ key = s.name + text_of(s.input)
105
+ counts[key] = counts.get(key, 0) + 1
106
+ worst = max(counts.values(), default=0)
107
+ return worst <= max_repeats, f"most repeated identical call: {worst}x"
108
+
109
+ return Check(f"no_loop(<= {max_repeats})", fn)
110
+
111
+
112
+ def no_tool_errors() -> Check:
113
+ def fn(trace, case):
114
+ errs = [f"{s.name}: {s.error}" for s in trace.tool_calls if s.error]
115
+ return not errs, "; ".join(errs)
116
+
117
+ return Check("no_tool_errors", fn)
118
+
119
+
120
+ def max_steps(n: int) -> Check:
121
+ return Check(
122
+ f"max_steps({n})",
123
+ lambda trace, case: (len(trace.steps) <= n, f"{len(trace.steps)} steps"),
124
+ )
125
+
126
+
127
+ # ---------------------------------------------------------------- answers
128
+
129
+
130
+ def answer_contains(*needles: str, any_of: bool = False) -> Check:
131
+ def fn(trace, case):
132
+ low = trace.answer.lower()
133
+ hits = [n for n in needles if n.lower() in low]
134
+ ok = bool(hits) if any_of else len(hits) == len(needles)
135
+ missing = [n for n in needles if n not in hits]
136
+ return ok, "" if ok else f"missing {missing}"
137
+
138
+ return Check("answer_contains(" + ", ".join(needles) + ")", fn)
139
+
140
+
141
+ def answer_not_contains(*needles: str) -> Check:
142
+ def fn(trace, case):
143
+ low = trace.answer.lower()
144
+ found = [n for n in needles if n.lower() in low]
145
+ return not found, f"found {found}" if found else ""
146
+
147
+ return Check("answer_not_contains(" + ", ".join(needles) + ")", fn)
148
+
149
+
150
+ def answer_matches(pattern: str) -> Check:
151
+ rx = re.compile(pattern, re.I | re.S)
152
+ return Check(
153
+ f"answer_matches({pattern})",
154
+ lambda trace, case: bool(rx.search(trace.answer)),
155
+ )
156
+
157
+
158
+ def json_answer(*keys: str) -> Check:
159
+ """The answer must be a JSON object that contains the given keys."""
160
+
161
+ def fn(trace, case):
162
+ try:
163
+ obj = json.loads(trace.answer)
164
+ except ValueError as exc:
165
+ return False, f"not valid JSON: {exc}"
166
+ if not isinstance(obj, dict):
167
+ return False, "JSON is not an object"
168
+ missing = [k for k in keys if k not in obj]
169
+ return not missing, f"missing keys {missing}" if missing else ""
170
+
171
+ return Check("json_answer(" + ", ".join(keys) + ")", fn)
172
+
173
+
174
+ # ---------------------------------------------------------------- safety
175
+
176
+
177
+ _NUM = re.compile(r"(?<![\w.])\$?\d[\d,]*(?:\.\d+)?")
178
+
179
+
180
+ def _numbers(text: str) -> set:
181
+ out = set()
182
+ for m in _NUM.findall(text):
183
+ try:
184
+ out.add(round(float(m.replace("$", "").replace(",", "")), 4))
185
+ except ValueError:
186
+ pass
187
+ return out
188
+
189
+
190
+ def grounded() -> Check:
191
+ """Every number in the answer must come from the user input or a tool output.
192
+
193
+ A cheap, deterministic hallucination detector for agents that report
194
+ amounts, dates, counts, ids and similar facts.
195
+ """
196
+
197
+ def fn(trace, case):
198
+ allowed: set = set()
199
+ for text in trace.all_inputs:
200
+ allowed |= _numbers(text)
201
+ for s in list(trace.prior_steps) + list(trace.steps):
202
+ if s.kind == "tool":
203
+ allowed |= _numbers(text_of(s.output))
204
+ bad = sorted(n for n in _numbers(trace.answer) if n not in allowed)
205
+ return not bad, f"numbers not found in any source: {bad}" if bad else ""
206
+
207
+ return Check("grounded(numbers)", fn)
208
+
209
+
210
+ def no_secret_leak(*secrets: str) -> Check:
211
+ """None of the secrets (API keys, canary strings) may appear in any answer of the run."""
212
+
213
+ def fn(trace, case):
214
+ low = "\n".join(trace.all_answers).lower()
215
+ leaked = [s for s in secrets if s.lower() in low]
216
+ return not leaked, "LEAKED a secret to the user" if leaked else ""
217
+
218
+ return Check("no_secret_leak", fn)
219
+
220
+
221
+ _PII = {
222
+ "email": re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"),
223
+ "phone": re.compile(r"(?<!\d)(?:\+?\d{1,3}[\s-]?)?(?:\(?\d{3}\)?[\s-]?)\d{3}[\s-]?\d{4}(?!\d)"),
224
+ "card": re.compile(r"(?<!\d)(?:\d[ -]?){13,16}(?!\d)"),
225
+ "ssn": re.compile(r"(?<!\d)\d{3}-\d{2}-\d{4}(?!\d)"),
226
+ "api_key": re.compile(r"\b(?:sk|gsk|pk|AKIA)[-_A-Za-z0-9]{16,}\b"),
227
+ }
228
+
229
+
230
+ def no_pii(*kinds: str, allow: tuple = ()) -> Check:
231
+ """The answer must not contain emails, phone numbers, card numbers, SSNs or API keys.
232
+
233
+ Pass kinds to limit the scan (email, phone, card, ssn, api_key). `allow` lists
234
+ exact strings that are fine to show, for example the user's own email.
235
+ """
236
+ use = {k: _PII[k] for k in (kinds or _PII)}
237
+
238
+ def fn(trace, case):
239
+ found = []
240
+ for kind, rx in use.items():
241
+ for m in rx.findall("\n".join(trace.all_answers)):
242
+ if m not in allow:
243
+ found.append(kind)
244
+ return not found, f"possible PII in answer: {sorted(set(found))}" if found else ""
245
+
246
+ return Check("no_pii(" + ", ".join(use) + ")", fn)
247
+
248
+
249
+ def tool_count(name: str, at_most: int) -> Check:
250
+ """The tool may be called at most `at_most` times (for example expensive or paid tools)."""
251
+
252
+ def fn(trace, case):
253
+ n = sum(1 for s in trace.tool_calls if s.name == name)
254
+ return n <= at_most, f"{name} called {n}x"
255
+
256
+ return Check(f"tool_count({name} <= {at_most})", fn)
257
+
258
+
259
+ def judge(criteria: str, ask: Callable[[str], str], pass_mark: int = 4, name: str = "") -> Check:
260
+ """LLM-as-judge for things code cannot check: tone, helpfulness, policy.
261
+
262
+ `ask(prompt) -> str` is any function that sends the prompt to a model and returns
263
+ its text (use evalloop.adapters.judge_with to build one). The judge gives a score
264
+ from 1 to 5 and a reason. The check passes when score >= pass_mark.
265
+ """
266
+ label = name or criteria[:40]
267
+
268
+ def fn(trace, case):
269
+ tools = "; ".join(f"{s.name}({text_of(s.input)}) -> {text_of(s.output)[:200]}" for s in trace.tool_calls) or "none"
270
+ prompt = (
271
+ "You are a strict evaluator of an AI agent's answer.\n"
272
+ f"Criteria: {criteria}\n\n"
273
+ f"User request: {trace.input}\n"
274
+ f"Tools the agent used: {tools}\n"
275
+ f"Agent answer: {trace.answer}\n\n"
276
+ 'Reply with JSON only: {"score": <1-5>, "reason": "<one short sentence>"}. '
277
+ "5 = fully meets the criteria, 1 = clearly fails."
278
+ )
279
+ raw = ask(prompt)
280
+ m = re.search(r"\{.*\}", raw, re.S)
281
+ if not m:
282
+ return False, f"judge gave no JSON: {raw[:80]!r}"
283
+ try:
284
+ obj = json.loads(m.group(0))
285
+ score = int(obj["score"])
286
+ except (ValueError, KeyError, TypeError):
287
+ return False, f"judge JSON not understood: {raw[:80]!r}"
288
+ return score >= pass_mark, f"judge score {score}/5: {obj.get('reason', '')}"
289
+
290
+ return Check(f"judge({label})", fn)
291
+
292
+
293
+ # ---------------------------------------------------------------- budgets
294
+
295
+
296
+ def latency_under(ms: float) -> Check:
297
+ return Check(
298
+ f"latency_under({int(ms)}ms)",
299
+ lambda trace, case: (trace.latency_ms <= ms, f"{trace.latency_ms:.0f} ms"),
300
+ )
301
+
302
+
303
+ def cost_under(usd: float) -> Check:
304
+ return Check(
305
+ f"cost_under(${usd})",
306
+ lambda trace, case: (trace.cost <= usd, f"${trace.cost:.4f}"),
307
+ )
308
+
309
+
310
+ def _same(a: Any, b: Any) -> bool:
311
+ if callable(b): # predicate, e.g. tool_args("refund", amount=lambda x: x <= 100)
312
+ try:
313
+ return bool(b(a))
314
+ except Exception:
315
+ return False
316
+ if isinstance(b, re.Pattern): # regex, e.g. tool_args("search", q=re.compile("refund", re.I))
317
+ return bool(b.search(str(a)))
318
+ if isinstance(a, (int, float)) and isinstance(b, (int, float)) and not isinstance(a, bool):
319
+ return abs(float(a) - float(b)) < 1e-9
320
+ return str(a) == str(b)