pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace_lab/clients.py ADDED
@@ -0,0 +1,568 @@
1
+ """pace_lab.clients: one logged, cached, resumable call layer for Ollama, Groq and Gemini (standard library only).
2
+
3
+ from pace_lab.clients import LLMClient, LLMRequest
4
+ client = LLMClient("outputs/pace_lab/run_001") # writes calls.jsonl and cache.jsonl there
5
+ res = client.call(LLMRequest(provider="ollama", model="llama3.1:8b", prompt="2+2?", temperature=0.0, seed=1,
6
+ max_tokens=64, item_id="i1", strategy="single", round=0, agent_id="a0"))
7
+ res.text, res.status
8
+
9
+ Design (all of it unit-tested with a fake transport; no network or key is needed to import or test):
10
+
11
+ * Transport abstraction. Every HTTP exchange goes through `Transport.request`; `UrllibTransport` is the real one, tests inject a fake.
12
+ * Providers. Ollama `/api/chat` (+ `/api/tags` for the model digest), Groq's OpenAI-compatible `/chat/completions` (custom User-Agent, key
13
+ from env GROQ_API_KEY), Gemini REST `models/<m>:generateContent` (key from env GEMINI_API_KEY, sent in the `x-goog-api-key` header, never in the URL).
14
+ * Retries with exponential backoff on timeouts, connection errors, HTTP 408/429/5xx (Retry-After honoured, capped); other 4xx fail at once.
15
+ * A per-provider rate limiter (requests per minute; `None` = unlimited). The default limits are conservative placeholders, not verified free-tier
16
+ numbers: set them from the provider's current documentation before a real run.
17
+ * Timeouts: 120 s per attempt by default.
18
+ * Log: one JSONL line per attempt in `calls.jsonl` (append-only). Fields: run_id, item_id, strategy, round, agent_id, timestamp, provider, model,
19
+ model_digest (Ollama only, else null), temperature, seed, top_p, max_tokens, prompt_tokens, completion_tokens, reasoning_tokens, reasoning_chars,
20
+ latency_s, text (raw), status (ok | timeout | error | cache_hit), attempt, cached, http_status, finish_reason, error, usage_raw, cache_key.
21
+ * Reasoning: `reasoning_tokens` is filled ONLY when the provider reports a count (Groq `completion_tokens_details.reasoning_tokens`, Gemini
22
+ `thoughtsTokenCount`); it is null otherwise and is never inferred. `reasoning_chars` is the length of separable thinking text (Ollama
23
+ `message.thinking`, a `<think>...</think>` block in the content, Groq/gpt-oss `message.reasoning`, Gemini thought parts), null when none is visible.
24
+ `completion_tokens` is whatever the provider reports; `usage_raw` keeps the provider's usage object so the convention can be checked later.
25
+ * Cache: `cache.jsonl` (append-only), key = sha256 of the JSON list [provider, model, prompt, temperature, seed, top_p, max_tokens]. A system
26
+ prompt and `extra` options are appended to that list only when non-empty, so the default key matches the specification exactly. Only status-ok
27
+ results are cached; a rerun makes no network call for a cached key (it writes a `cache_hit` log line). The cache is read at start-up, so a
28
+ killed run resumes where it stopped. The model digest is NOT in the key (by specification): if a local model is re-pulled, delete the cache.
29
+ * Secrets: keys are read from os.environ at call time (or from a gitignored .env file via `load_dotenv_file`, which the caller must invoke
30
+ explicitly). They are never written to the log, cache, exceptions or reprs; every string that leaves this module is passed through a Redactor
31
+ that removes the exact key values and common key patterns.
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import hashlib
36
+ import json
37
+ import os
38
+ import random
39
+ import re
40
+ import socket
41
+ import threading
42
+ import time
43
+ import urllib.error
44
+ import urllib.request
45
+ import uuid
46
+ from dataclasses import dataclass, field
47
+ from datetime import datetime, timezone
48
+ from typing import Any, Callable, Dict, List, Optional, Tuple
49
+
50
+ DEFAULT_TIMEOUT = 120.0
51
+ SECRET_ENV_VARS = ("GROQ_API_KEY", "GEMINI_API_KEY")
52
+ DEFAULT_USER_AGENT = "pace-lab/0.1 (+research-harness; python-urllib)"
53
+ RETRY_STATUSES = frozenset({408, 429, 500, 502, 503, 504})
54
+ # Conservative placeholders (requests per minute). NOT verified provider limits.
55
+ DEFAULT_RPM: Dict[str, Optional[float]] = {"ollama": None, "groq": 30.0, "gemini": 10.0}
56
+
57
+
58
+ # ----------------------------------------------------------------------------- errors and redaction
59
+ class ClientError(Exception):
60
+ """Base error. Messages are always redacted before the exception is built."""
61
+
62
+
63
+ class MissingKeyError(ClientError):
64
+ """An API key environment variable is not set (the message names the variable, never a value)."""
65
+
66
+
67
+ class TransportError(Exception):
68
+ pass
69
+
70
+
71
+ class TransportTimeout(TransportError):
72
+ pass
73
+
74
+
75
+ _PATTERNS = [
76
+ re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-~+/=]{8,}"),
77
+ re.compile(r"AIza[0-9A-Za-z_\-]{20,}"),
78
+ re.compile(r"gsk_[A-Za-z0-9]{10,}"),
79
+ re.compile(r"(?i)((?:api[_-]?key|key|token)=)[^&\s\"']+"),
80
+ ]
81
+
82
+
83
+ class Redactor:
84
+ """Replace known secret values and common key shapes by [REDACTED]. Secrets shorter than 4 characters are ignored."""
85
+
86
+ def __init__(self, secrets=()):
87
+ self._secrets: List[str] = []
88
+ for s in secrets:
89
+ self.add(s)
90
+
91
+ def add(self, secret: Optional[str]) -> None:
92
+ if not secret or len(secret) < 4:
93
+ return
94
+ for variant in (secret, json.dumps(secret, ensure_ascii=False)[1:-1], json.dumps(secret)[1:-1]):
95
+ if variant and variant not in self._secrets:
96
+ self._secrets.append(variant)
97
+ self._secrets.sort(key=len, reverse=True)
98
+
99
+ def __call__(self, text: Any) -> str:
100
+ s = text if isinstance(text, str) else str(text)
101
+ for sec in self._secrets:
102
+ s = s.replace(sec, "[REDACTED]")
103
+ s = _PATTERNS[0].sub(r"\1[REDACTED]", s)
104
+ s = _PATTERNS[1].sub("[REDACTED]", s)
105
+ s = _PATTERNS[2].sub("[REDACTED]", s)
106
+ s = _PATTERNS[3].sub(r"\1[REDACTED]", s)
107
+ return s
108
+
109
+ def __repr__(self) -> str: # never show the secrets
110
+ return "Redactor(n_secrets=%d)" % len(self._secrets)
111
+
112
+
113
+ def load_dotenv_file(path: str, environ: Optional[Dict[str, str]] = None) -> List[str]:
114
+ """Minimal .env reader (KEY=VALUE lines). Sets variables that are not already set; returns the NAMES it set (never the values).
115
+ Opt-in: nothing in this module calls it automatically."""
116
+ env = os.environ if environ is None else environ
117
+ names: List[str] = []
118
+ if not os.path.isfile(path):
119
+ return names
120
+ with open(path, encoding="utf-8") as fh:
121
+ for raw in fh:
122
+ line = raw.strip()
123
+ if not line or line.startswith("#") or "=" not in line:
124
+ continue
125
+ k, v = line.split("=", 1)
126
+ k = k.strip()
127
+ if k.startswith("export "):
128
+ k = k[7:].strip()
129
+ v = v.strip()
130
+ if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
131
+ v = v[1:-1]
132
+ if k and k not in env:
133
+ env[k] = v
134
+ names.append(k)
135
+ return names
136
+
137
+
138
+ # ----------------------------------------------------------------------------- transport
139
+ @dataclass
140
+ class HttpResponse:
141
+ status: int
142
+ body: str
143
+ headers: Dict[str, str] = field(default_factory=dict)
144
+
145
+
146
+ class Transport:
147
+ def request(self, method: str, url: str, headers: Dict[str, str], body: Optional[bytes], timeout: float) -> HttpResponse:
148
+ raise NotImplementedError
149
+
150
+
151
+ class UrllibTransport(Transport):
152
+ """Real HTTP through urllib. Error messages never include the URL or headers."""
153
+
154
+ def request(self, method, url, headers, body, timeout):
155
+ req = urllib.request.Request(url, data=body, headers=dict(headers), method=method)
156
+ try:
157
+ with urllib.request.urlopen(req, timeout=timeout) as r:
158
+ return HttpResponse(r.status, r.read().decode("utf-8", "replace"), {k.lower(): v for k, v in r.headers.items()})
159
+ except urllib.error.HTTPError as e:
160
+ try:
161
+ text = e.read().decode("utf-8", "replace")
162
+ except Exception:
163
+ text = ""
164
+ return HttpResponse(e.code, text, {k.lower(): v for k, v in (e.headers or {}).items()})
165
+ except (socket.timeout, TimeoutError):
166
+ raise TransportTimeout("timed out after %.0f s" % timeout) from None
167
+ except urllib.error.URLError as e:
168
+ if isinstance(e.reason, (socket.timeout, TimeoutError)):
169
+ raise TransportTimeout("timed out after %.0f s" % timeout) from None
170
+ raise TransportError("connection error: %s" % type(e.reason).__name__) from None
171
+ except OSError as e:
172
+ raise TransportError("connection error: %s" % type(e).__name__) from None
173
+
174
+
175
+ # ----------------------------------------------------------------------------- rate limiter
176
+ class RateLimiter:
177
+ """Spaces calls at least 60 / requests_per_minute seconds apart (thread-safe, reservation style). None or 0 = unlimited."""
178
+
179
+ def __init__(self, requests_per_minute: Optional[float] = None, clock: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep):
180
+ self.interval = 60.0 / requests_per_minute if requests_per_minute else 0.0
181
+ self._clock, self._sleep = clock, sleep
182
+ self._next = 0.0
183
+ self._lock = threading.Lock()
184
+
185
+ def acquire(self) -> float:
186
+ """Block until a call may start; returns the seconds waited."""
187
+ if not self.interval:
188
+ return 0.0
189
+ with self._lock:
190
+ now = self._clock()
191
+ start = max(now, self._next)
192
+ self._next = start + self.interval
193
+ wait = start - now
194
+ if wait > 0:
195
+ self._sleep(wait)
196
+ return max(wait, 0.0)
197
+
198
+
199
+ # ----------------------------------------------------------------------------- request / result
200
+ @dataclass
201
+ class LLMRequest:
202
+ provider: str # "ollama" | "groq" | "gemini"
203
+ model: str
204
+ prompt: str
205
+ system: Optional[str] = None
206
+ temperature: float = 0.0
207
+ seed: Optional[int] = None
208
+ top_p: float = 1.0
209
+ max_tokens: int = 2048
210
+ extra: Dict[str, Any] = field(default_factory=dict) # provider options, e.g. {"think": True} (Ollama), {"reasoning_effort": "low"} (Groq)
211
+ # labels, not part of the cache key
212
+ run_id: Optional[str] = None
213
+ item_id: Optional[str] = None
214
+ strategy: Optional[str] = None
215
+ round: Optional[int] = None
216
+ agent_id: Optional[str] = None
217
+
218
+
219
+ @dataclass
220
+ class Parsed:
221
+ text: str
222
+ prompt_tokens: Optional[int] = None
223
+ completion_tokens: Optional[int] = None
224
+ reasoning_tokens: Optional[int] = None
225
+ reasoning_chars: Optional[int] = None
226
+ finish_reason: Optional[str] = None
227
+ usage_raw: Optional[dict] = None
228
+
229
+
230
+ @dataclass
231
+ class LLMResult:
232
+ status: str # ok | timeout | error
233
+ text: Optional[str]
234
+ record: dict # the log line of the final attempt
235
+ cached: bool = False
236
+ error: Optional[str] = None
237
+
238
+ @property
239
+ def ok(self) -> bool:
240
+ return self.status == "ok"
241
+
242
+ def raise_for_status(self) -> None:
243
+ if not self.ok:
244
+ raise ClientError("call failed: status=%s error=%s" % (self.status, self.error))
245
+
246
+
247
+ class ParseError(Exception):
248
+ pass
249
+
250
+
251
+ def cache_key(req: LLMRequest) -> str:
252
+ """sha256 over the JSON list [provider, model, prompt, temperature, seed, top_p, max_tokens] (+ system, extra only when set)."""
253
+ parts: List[Any] = [req.provider, req.model, req.prompt, req.temperature, req.seed, req.top_p, req.max_tokens]
254
+ if req.system:
255
+ parts.append({"system": req.system})
256
+ if req.extra:
257
+ parts.append({"extra": req.extra})
258
+ return hashlib.sha256(json.dumps(parts, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest()
259
+
260
+
261
+ _THINK_CLOSED = re.compile(r"<think>(.*?)</think>", re.S)
262
+
263
+
264
+ def think_chars(text: str) -> Optional[int]:
265
+ """Characters inside <think> blocks of a response (an unclosed trailing block counts to the end); None when the text has no <think> tag."""
266
+ if not text or "<think>" not in text:
267
+ return None
268
+ total = sum(len(m) for m in _THINK_CLOSED.findall(text))
269
+ rest = _THINK_CLOSED.sub("", text)
270
+ if "<think>" in rest:
271
+ total += len(rest.split("<think>", 1)[1])
272
+ return total
273
+
274
+
275
+ def _int(x: Any) -> Optional[int]:
276
+ return x if isinstance(x, int) and not isinstance(x, bool) else None
277
+
278
+
279
+ # ----------------------------------------------------------------------------- providers
280
+ class Provider:
281
+ name = ""
282
+ key_env: Optional[str] = None
283
+
284
+ def __init__(self, base_url: str, user_agent: str = DEFAULT_USER_AGENT):
285
+ self.base_url = base_url.rstrip("/")
286
+ self.user_agent = user_agent
287
+
288
+ def build(self, req: LLMRequest, api_key: Optional[str]) -> Tuple[str, str, Dict[str, str], bytes]:
289
+ raise NotImplementedError
290
+
291
+ def parse(self, data: dict) -> Parsed:
292
+ raise NotImplementedError
293
+
294
+ @staticmethod
295
+ def _messages(req: LLMRequest) -> List[dict]:
296
+ msgs = []
297
+ if req.system:
298
+ msgs.append({"role": "system", "content": req.system})
299
+ msgs.append({"role": "user", "content": req.prompt})
300
+ return msgs
301
+
302
+
303
+ class OllamaProvider(Provider):
304
+ name = "ollama"
305
+
306
+ def build(self, req, api_key):
307
+ opts: Dict[str, Any] = {"temperature": req.temperature, "top_p": req.top_p, "num_predict": req.max_tokens}
308
+ if req.seed is not None:
309
+ opts["seed"] = req.seed
310
+ body: Dict[str, Any] = {"model": req.model, "messages": self._messages(req), "stream": False}
311
+ extra = dict(req.extra)
312
+ opts.update(extra.pop("options", {}) or {})
313
+ body.update(extra) # e.g. think, keep_alive
314
+ body["options"] = opts
315
+ return "POST", self.base_url + "/api/chat", {"Content-Type": "application/json", "User-Agent": self.user_agent}, json.dumps(body).encode("utf-8")
316
+
317
+ def parse(self, data):
318
+ msg = data.get("message")
319
+ if not isinstance(msg, dict) or "content" not in msg:
320
+ raise ParseError("no message.content in the Ollama response")
321
+ text = msg.get("content") or ""
322
+ thinking = msg.get("thinking")
323
+ rc = len(thinking) if isinstance(thinking, str) else think_chars(text)
324
+ usage = {k: data[k] for k in ("prompt_eval_count", "eval_count", "total_duration", "load_duration", "prompt_eval_duration", "eval_duration") if k in data}
325
+ return Parsed(text=text, prompt_tokens=_int(data.get("prompt_eval_count")), completion_tokens=_int(data.get("eval_count")),
326
+ reasoning_tokens=None, reasoning_chars=rc, finish_reason=data.get("done_reason"), usage_raw=usage or None)
327
+
328
+
329
+ class GroqProvider(Provider):
330
+ name = "groq"
331
+ key_env = "GROQ_API_KEY"
332
+
333
+ def build(self, req, api_key):
334
+ body: Dict[str, Any] = {"model": req.model, "messages": self._messages(req), "temperature": req.temperature, "top_p": req.top_p,
335
+ "max_completion_tokens": req.max_tokens}
336
+ if req.seed is not None:
337
+ body["seed"] = req.seed
338
+ body.update(req.extra)
339
+ headers = {"Content-Type": "application/json", "Authorization": "Bearer " + str(api_key), "User-Agent": self.user_agent}
340
+ return "POST", self.base_url + "/chat/completions", headers, json.dumps(body).encode("utf-8")
341
+
342
+ def parse(self, data):
343
+ try:
344
+ choice = data["choices"][0]
345
+ msg = choice["message"]
346
+ except (KeyError, IndexError, TypeError):
347
+ raise ParseError("no choices[0].message in the Groq response") from None
348
+ text = msg.get("content") or ""
349
+ reasoning = msg.get("reasoning")
350
+ rc = len(reasoning) if isinstance(reasoning, str) else think_chars(text)
351
+ usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
352
+ details = usage.get("completion_tokens_details") if isinstance(usage.get("completion_tokens_details"), dict) else {}
353
+ return Parsed(text=text, prompt_tokens=_int(usage.get("prompt_tokens")), completion_tokens=_int(usage.get("completion_tokens")),
354
+ reasoning_tokens=_int(details.get("reasoning_tokens")), reasoning_chars=rc, finish_reason=choice.get("finish_reason"),
355
+ usage_raw=usage or None)
356
+
357
+
358
+ class GeminiProvider(Provider):
359
+ name = "gemini"
360
+ key_env = "GEMINI_API_KEY"
361
+
362
+ def build(self, req, api_key):
363
+ gen: Dict[str, Any] = {"temperature": req.temperature, "topP": req.top_p, "maxOutputTokens": req.max_tokens}
364
+ if req.seed is not None:
365
+ gen["seed"] = req.seed
366
+ extra = dict(req.extra)
367
+ gen.update(extra.pop("generationConfig", {}) or {})
368
+ body: Dict[str, Any] = {"contents": [{"role": "user", "parts": [{"text": req.prompt}]}], "generationConfig": gen}
369
+ if req.system:
370
+ body["systemInstruction"] = {"parts": [{"text": req.system}]}
371
+ body.update(extra)
372
+ headers = {"Content-Type": "application/json", "x-goog-api-key": str(api_key), "User-Agent": self.user_agent}
373
+ return "POST", "%s/models/%s:generateContent" % (self.base_url, req.model), headers, json.dumps(body).encode("utf-8")
374
+
375
+ def parse(self, data):
376
+ cands = data.get("candidates")
377
+ if not cands:
378
+ fb = data.get("promptFeedback") or {}
379
+ raise ParseError("no candidates (blockReason=%s)" % fb.get("blockReason"))
380
+ cand = cands[0]
381
+ parts = ((cand.get("content") or {}).get("parts")) or []
382
+ text = "".join(p.get("text", "") for p in parts if isinstance(p, dict) and not p.get("thought"))
383
+ thought = [p.get("text", "") for p in parts if isinstance(p, dict) and p.get("thought")]
384
+ um = data.get("usageMetadata") if isinstance(data.get("usageMetadata"), dict) else {}
385
+ return Parsed(text=text, prompt_tokens=_int(um.get("promptTokenCount")), completion_tokens=_int(um.get("candidatesTokenCount")),
386
+ reasoning_tokens=_int(um.get("thoughtsTokenCount")), reasoning_chars=sum(len(t) for t in thought) if thought else None,
387
+ finish_reason=cand.get("finishReason"), usage_raw=um or None)
388
+
389
+
390
+ # ----------------------------------------------------------------------------- the client
391
+ def _utc_iso(ts: float) -> str:
392
+ return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat(timespec="milliseconds")
393
+
394
+
395
+ class LLMClient:
396
+ def __init__(self, log_dir: str, run_id: Optional[str] = None, transport: Optional[Transport] = None,
397
+ rate_limits: Optional[Dict[str, Optional[float]]] = None, max_retries: int = 4, backoff_base: float = 1.0, backoff_max: float = 60.0,
398
+ timeout: float = DEFAULT_TIMEOUT, ollama_url: str = "http://localhost:11434", groq_url: str = "https://api.groq.com/openai/v1",
399
+ gemini_url: str = "https://generativelanguage.googleapis.com/v1beta", user_agent: str = DEFAULT_USER_AGENT,
400
+ environ: Optional[Dict[str, str]] = None, extra_secrets=(), sleep: Callable[[float], None] = time.sleep,
401
+ monotonic: Callable[[], float] = time.monotonic, wall_clock: Callable[[], float] = time.time,
402
+ rng: Optional[random.Random] = None, log_name: str = "calls.jsonl", cache_name: str = "cache.jsonl"):
403
+ os.makedirs(log_dir, exist_ok=True)
404
+ self.log_path = os.path.join(log_dir, log_name)
405
+ self.cache_path = os.path.join(log_dir, cache_name)
406
+ self.run_id = run_id or (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:6])
407
+ self.transport = transport or UrllibTransport()
408
+ self.max_retries, self.backoff_base, self.backoff_max, self.timeout = max_retries, backoff_base, backoff_max, timeout
409
+ self._environ = os.environ if environ is None else environ
410
+ self._sleep, self._wall = sleep, wall_clock
411
+ self._mono = monotonic
412
+ self._rng = rng or random.Random()
413
+ self.providers: Dict[str, Provider] = {"ollama": OllamaProvider(ollama_url, user_agent), "groq": GroqProvider(groq_url, user_agent),
414
+ "gemini": GeminiProvider(gemini_url, user_agent)}
415
+ limits = dict(DEFAULT_RPM)
416
+ limits.update(rate_limits or {})
417
+ self.limiters = {p: RateLimiter(limits.get(p), clock=monotonic, sleep=sleep) for p in self.providers}
418
+ self.redact = Redactor(extra_secrets)
419
+ for var in SECRET_ENV_VARS:
420
+ self.redact.add(self._environ.get(var))
421
+ self._io_lock = threading.Lock()
422
+ self._key_locks: Dict[str, threading.Lock] = {}
423
+ self._digests: Dict[str, Optional[str]] = {}
424
+ self._cache: Dict[str, dict] = {}
425
+ self._load_cache()
426
+
427
+ def __repr__(self) -> str:
428
+ return "LLMClient(run_id=%r, log=%r)" % (self.run_id, self.log_path)
429
+
430
+ # --- persistence
431
+ def _load_cache(self) -> None:
432
+ if not os.path.isfile(self.cache_path):
433
+ return
434
+ with open(self.cache_path, encoding="utf-8") as fh:
435
+ for line in fh:
436
+ line = line.strip()
437
+ if not line:
438
+ continue
439
+ try:
440
+ rec = json.loads(line)
441
+ except ValueError:
442
+ continue # a half-written last line after a crash
443
+ if isinstance(rec, dict) and rec.get("cache_key") and rec.get("status") == "ok":
444
+ self._cache.setdefault(rec["cache_key"], rec)
445
+
446
+ def _append(self, path: str, rec: dict) -> None:
447
+ line = self.redact(json.dumps(rec, ensure_ascii=False))
448
+ with self._io_lock:
449
+ with open(path, "a", encoding="utf-8", newline="\n") as fh:
450
+ fh.write(line + "\n")
451
+ fh.flush()
452
+
453
+ @property
454
+ def cache_size(self) -> int:
455
+ return len(self._cache)
456
+
457
+ # --- helpers
458
+ def _api_key(self, provider: Provider) -> Optional[str]:
459
+ if not provider.key_env:
460
+ return None
461
+ val = self._environ.get(provider.key_env)
462
+ if not val:
463
+ raise MissingKeyError("environment variable %s is not set" % provider.key_env)
464
+ self.redact.add(val)
465
+ return val
466
+
467
+ def model_digest(self, model: str) -> Optional[str]:
468
+ """Digest of a local Ollama model from /api/tags (None when the daemon or the model is not found). Fetched once per model."""
469
+ with self._io_lock:
470
+ if model in self._digests:
471
+ return self._digests[model]
472
+ digest = None
473
+ try:
474
+ r = self.transport.request("GET", self.providers["ollama"].base_url + "/api/tags", {"User-Agent": DEFAULT_USER_AGENT}, None, min(self.timeout, 30.0))
475
+ if r.status == 200:
476
+ models = json.loads(r.body).get("models", [])
477
+ wanted = {model, model if ":" in model else model + ":latest"}
478
+ for m in models:
479
+ if m.get("name") in wanted or m.get("model") in wanted:
480
+ digest = m.get("digest")
481
+ break
482
+ except Exception:
483
+ digest = None
484
+ with self._io_lock:
485
+ self._digests[model] = digest
486
+ return digest
487
+
488
+ def _line(self, req: LLMRequest, key: str, attempt: int, status: str, digest: Optional[str], parsed: Optional[Parsed] = None,
489
+ latency: Optional[float] = None, http_status: Optional[int] = None, error: Optional[str] = None, cached: bool = False) -> dict:
490
+ p = parsed
491
+ return {
492
+ "run_id": req.run_id or self.run_id, "item_id": req.item_id, "strategy": req.strategy, "round": req.round, "agent_id": req.agent_id,
493
+ "timestamp": _utc_iso(self._wall()), "provider": req.provider, "model": req.model, "model_digest": digest,
494
+ "temperature": req.temperature, "seed": req.seed, "top_p": req.top_p, "max_tokens": req.max_tokens,
495
+ "prompt_tokens": p.prompt_tokens if p else None, "completion_tokens": p.completion_tokens if p else None,
496
+ "reasoning_tokens": p.reasoning_tokens if p else None, "reasoning_chars": p.reasoning_chars if p else None,
497
+ "latency_s": latency, "text": p.text if p else None, "status": status, "attempt": attempt, "cached": cached,
498
+ "http_status": http_status, "finish_reason": p.finish_reason if p else None, "error": error,
499
+ "usage_raw": p.usage_raw if p else None, "cache_key": key,
500
+ }
501
+
502
+ def _delay(self, attempt: int, resp: Optional[HttpResponse]) -> float:
503
+ if resp is not None:
504
+ ra = resp.headers.get("retry-after")
505
+ if ra:
506
+ try:
507
+ return min(self.backoff_max, max(0.0, float(ra)))
508
+ except ValueError:
509
+ pass
510
+ return min(self.backoff_max, self.backoff_base * (2 ** (attempt - 1)) * (1.0 + 0.25 * self._rng.random()))
511
+
512
+ # --- the call
513
+ def call(self, req: LLMRequest) -> LLMResult:
514
+ if req.provider not in self.providers:
515
+ raise ClientError("unknown provider %r (expected one of %s)" % (req.provider, sorted(self.providers)))
516
+ key = cache_key(req)
517
+ with self._io_lock:
518
+ lock = self._key_locks.setdefault(key, threading.Lock())
519
+ with lock:
520
+ hit = self._cache.get(key)
521
+ if hit is not None:
522
+ p = Parsed(text=hit.get("text") or "", prompt_tokens=hit.get("prompt_tokens"), completion_tokens=hit.get("completion_tokens"),
523
+ reasoning_tokens=hit.get("reasoning_tokens"), reasoning_chars=hit.get("reasoning_chars"), finish_reason=hit.get("finish_reason"),
524
+ usage_raw=hit.get("usage_raw"))
525
+ rec = self._line(req, key, 0, "cache_hit", hit.get("model_digest"), p, None, hit.get("http_status"), None, True)
526
+ self._append(self.log_path, rec)
527
+ return LLMResult("ok", p.text, rec, cached=True)
528
+ provider = self.providers[req.provider]
529
+ api_key = self._api_key(provider)
530
+ digest = self.model_digest(req.model) if req.provider == "ollama" else None
531
+ method, url, headers, body = provider.build(req, api_key)
532
+ last: Optional[LLMResult] = None
533
+ for attempt in range(1, self.max_retries + 2):
534
+ self.limiters[req.provider].acquire()
535
+ t0 = self._mono()
536
+ resp: Optional[HttpResponse] = None
537
+ status, err, retry, parsed = "error", None, False, None
538
+ try:
539
+ resp = self.transport.request(method, url, headers, body, self.timeout)
540
+ except TransportTimeout as e:
541
+ status, err, retry = "timeout", self.redact(e), True
542
+ except TransportError as e:
543
+ status, err, retry = "error", self.redact(e), True
544
+ except Exception as e: # a misbehaving transport must not leak its message (it may contain the URL or headers)
545
+ status, err, retry = "error", self.redact("%s: %s" % (type(e).__name__, e)), False
546
+ latency = self._mono() - t0
547
+ http_status = resp.status if resp is not None else None
548
+ if resp is not None:
549
+ if resp.status == 200:
550
+ try:
551
+ parsed = provider.parse(json.loads(resp.body))
552
+ status = "ok"
553
+ except (ParseError, ValueError, TypeError, AttributeError) as e:
554
+ status, err = "error", self.redact("unparseable response: %s" % e)
555
+ else:
556
+ status, err = "error", self.redact("HTTP %d: %s" % (resp.status, resp.body[:300].replace("\n", " ")))
557
+ retry = resp.status in RETRY_STATUSES
558
+ rec = self._line(req, key, attempt, status, digest, parsed, round(latency, 6), http_status, err)
559
+ self._append(self.log_path, rec)
560
+ last = LLMResult(status, parsed.text if parsed else None, rec, False, err)
561
+ if status == "ok":
562
+ self._append(self.cache_path, rec)
563
+ self._cache[key] = json.loads(self.redact(json.dumps(rec, ensure_ascii=False)))
564
+ return last
565
+ if not retry or attempt > self.max_retries:
566
+ return last
567
+ self._sleep(self._delay(attempt, resp))
568
+ return last # pragma: no cover