pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace_lab/clients.py
ADDED
|
@@ -0,0 +1,568 @@
|
|
|
1
|
+
"""pace_lab.clients: one logged, cached, resumable call layer for Ollama, Groq and Gemini (standard library only).
|
|
2
|
+
|
|
3
|
+
from pace_lab.clients import LLMClient, LLMRequest
|
|
4
|
+
client = LLMClient("outputs/pace_lab/run_001") # writes calls.jsonl and cache.jsonl there
|
|
5
|
+
res = client.call(LLMRequest(provider="ollama", model="llama3.1:8b", prompt="2+2?", temperature=0.0, seed=1,
|
|
6
|
+
max_tokens=64, item_id="i1", strategy="single", round=0, agent_id="a0"))
|
|
7
|
+
res.text, res.status
|
|
8
|
+
|
|
9
|
+
Design (all of it unit-tested with a fake transport; no network or key is needed to import or test):
|
|
10
|
+
|
|
11
|
+
* Transport abstraction. Every HTTP exchange goes through `Transport.request`; `UrllibTransport` is the real one, tests inject a fake.
|
|
12
|
+
* Providers. Ollama `/api/chat` (+ `/api/tags` for the model digest), Groq's OpenAI-compatible `/chat/completions` (custom User-Agent, key
|
|
13
|
+
from env GROQ_API_KEY), Gemini REST `models/<m>:generateContent` (key from env GEMINI_API_KEY, sent in the `x-goog-api-key` header, never in the URL).
|
|
14
|
+
* Retries with exponential backoff on timeouts, connection errors, HTTP 408/429/5xx (Retry-After honoured, capped); other 4xx fail at once.
|
|
15
|
+
* A per-provider rate limiter (requests per minute; `None` = unlimited). The default limits are conservative placeholders, not verified free-tier
|
|
16
|
+
numbers: set them from the provider's current documentation before a real run.
|
|
17
|
+
* Timeouts: 120 s per attempt by default.
|
|
18
|
+
* Log: one JSONL line per attempt in `calls.jsonl` (append-only). Fields: run_id, item_id, strategy, round, agent_id, timestamp, provider, model,
|
|
19
|
+
model_digest (Ollama only, else null), temperature, seed, top_p, max_tokens, prompt_tokens, completion_tokens, reasoning_tokens, reasoning_chars,
|
|
20
|
+
latency_s, text (raw), status (ok | timeout | error | cache_hit), attempt, cached, http_status, finish_reason, error, usage_raw, cache_key.
|
|
21
|
+
* Reasoning: `reasoning_tokens` is filled ONLY when the provider reports a count (Groq `completion_tokens_details.reasoning_tokens`, Gemini
|
|
22
|
+
`thoughtsTokenCount`); it is null otherwise and is never inferred. `reasoning_chars` is the length of separable thinking text (Ollama
|
|
23
|
+
`message.thinking`, a `<think>...</think>` block in the content, Groq/gpt-oss `message.reasoning`, Gemini thought parts), null when none is visible.
|
|
24
|
+
`completion_tokens` is whatever the provider reports; `usage_raw` keeps the provider's usage object so the convention can be checked later.
|
|
25
|
+
* Cache: `cache.jsonl` (append-only), key = sha256 of the JSON list [provider, model, prompt, temperature, seed, top_p, max_tokens]. A system
|
|
26
|
+
prompt and `extra` options are appended to that list only when non-empty, so the default key matches the specification exactly. Only status-ok
|
|
27
|
+
results are cached; a rerun makes no network call for a cached key (it writes a `cache_hit` log line). The cache is read at start-up, so a
|
|
28
|
+
killed run resumes where it stopped. The model digest is NOT in the key (by specification): if a local model is re-pulled, delete the cache.
|
|
29
|
+
* Secrets: keys are read from os.environ at call time (or from a gitignored .env file via `load_dotenv_file`, which the caller must invoke
|
|
30
|
+
explicitly). They are never written to the log, cache, exceptions or reprs; every string that leaves this module is passed through a Redactor
|
|
31
|
+
that removes the exact key values and common key patterns.
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import hashlib
|
|
36
|
+
import json
|
|
37
|
+
import os
|
|
38
|
+
import random
|
|
39
|
+
import re
|
|
40
|
+
import socket
|
|
41
|
+
import threading
|
|
42
|
+
import time
|
|
43
|
+
import urllib.error
|
|
44
|
+
import urllib.request
|
|
45
|
+
import uuid
|
|
46
|
+
from dataclasses import dataclass, field
|
|
47
|
+
from datetime import datetime, timezone
|
|
48
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
49
|
+
|
|
50
|
+
DEFAULT_TIMEOUT = 120.0
|
|
51
|
+
SECRET_ENV_VARS = ("GROQ_API_KEY", "GEMINI_API_KEY")
|
|
52
|
+
DEFAULT_USER_AGENT = "pace-lab/0.1 (+research-harness; python-urllib)"
|
|
53
|
+
RETRY_STATUSES = frozenset({408, 429, 500, 502, 503, 504})
|
|
54
|
+
# Conservative placeholders (requests per minute). NOT verified provider limits.
|
|
55
|
+
DEFAULT_RPM: Dict[str, Optional[float]] = {"ollama": None, "groq": 30.0, "gemini": 10.0}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ----------------------------------------------------------------------------- errors and redaction
|
|
59
|
+
class ClientError(Exception):
|
|
60
|
+
"""Base error. Messages are always redacted before the exception is built."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class MissingKeyError(ClientError):
|
|
64
|
+
"""An API key environment variable is not set (the message names the variable, never a value)."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class TransportError(Exception):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class TransportTimeout(TransportError):
|
|
72
|
+
pass
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
_PATTERNS = [
|
|
76
|
+
re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-~+/=]{8,}"),
|
|
77
|
+
re.compile(r"AIza[0-9A-Za-z_\-]{20,}"),
|
|
78
|
+
re.compile(r"gsk_[A-Za-z0-9]{10,}"),
|
|
79
|
+
re.compile(r"(?i)((?:api[_-]?key|key|token)=)[^&\s\"']+"),
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class Redactor:
|
|
84
|
+
"""Replace known secret values and common key shapes by [REDACTED]. Secrets shorter than 4 characters are ignored."""
|
|
85
|
+
|
|
86
|
+
def __init__(self, secrets=()):
|
|
87
|
+
self._secrets: List[str] = []
|
|
88
|
+
for s in secrets:
|
|
89
|
+
self.add(s)
|
|
90
|
+
|
|
91
|
+
def add(self, secret: Optional[str]) -> None:
|
|
92
|
+
if not secret or len(secret) < 4:
|
|
93
|
+
return
|
|
94
|
+
for variant in (secret, json.dumps(secret, ensure_ascii=False)[1:-1], json.dumps(secret)[1:-1]):
|
|
95
|
+
if variant and variant not in self._secrets:
|
|
96
|
+
self._secrets.append(variant)
|
|
97
|
+
self._secrets.sort(key=len, reverse=True)
|
|
98
|
+
|
|
99
|
+
def __call__(self, text: Any) -> str:
|
|
100
|
+
s = text if isinstance(text, str) else str(text)
|
|
101
|
+
for sec in self._secrets:
|
|
102
|
+
s = s.replace(sec, "[REDACTED]")
|
|
103
|
+
s = _PATTERNS[0].sub(r"\1[REDACTED]", s)
|
|
104
|
+
s = _PATTERNS[1].sub("[REDACTED]", s)
|
|
105
|
+
s = _PATTERNS[2].sub("[REDACTED]", s)
|
|
106
|
+
s = _PATTERNS[3].sub(r"\1[REDACTED]", s)
|
|
107
|
+
return s
|
|
108
|
+
|
|
109
|
+
def __repr__(self) -> str: # never show the secrets
|
|
110
|
+
return "Redactor(n_secrets=%d)" % len(self._secrets)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def load_dotenv_file(path: str, environ: Optional[Dict[str, str]] = None) -> List[str]:
|
|
114
|
+
"""Minimal .env reader (KEY=VALUE lines). Sets variables that are not already set; returns the NAMES it set (never the values).
|
|
115
|
+
Opt-in: nothing in this module calls it automatically."""
|
|
116
|
+
env = os.environ if environ is None else environ
|
|
117
|
+
names: List[str] = []
|
|
118
|
+
if not os.path.isfile(path):
|
|
119
|
+
return names
|
|
120
|
+
with open(path, encoding="utf-8") as fh:
|
|
121
|
+
for raw in fh:
|
|
122
|
+
line = raw.strip()
|
|
123
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
124
|
+
continue
|
|
125
|
+
k, v = line.split("=", 1)
|
|
126
|
+
k = k.strip()
|
|
127
|
+
if k.startswith("export "):
|
|
128
|
+
k = k[7:].strip()
|
|
129
|
+
v = v.strip()
|
|
130
|
+
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
|
|
131
|
+
v = v[1:-1]
|
|
132
|
+
if k and k not in env:
|
|
133
|
+
env[k] = v
|
|
134
|
+
names.append(k)
|
|
135
|
+
return names
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ----------------------------------------------------------------------------- transport
|
|
139
|
+
@dataclass
|
|
140
|
+
class HttpResponse:
|
|
141
|
+
status: int
|
|
142
|
+
body: str
|
|
143
|
+
headers: Dict[str, str] = field(default_factory=dict)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class Transport:
|
|
147
|
+
def request(self, method: str, url: str, headers: Dict[str, str], body: Optional[bytes], timeout: float) -> HttpResponse:
|
|
148
|
+
raise NotImplementedError
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class UrllibTransport(Transport):
|
|
152
|
+
"""Real HTTP through urllib. Error messages never include the URL or headers."""
|
|
153
|
+
|
|
154
|
+
def request(self, method, url, headers, body, timeout):
|
|
155
|
+
req = urllib.request.Request(url, data=body, headers=dict(headers), method=method)
|
|
156
|
+
try:
|
|
157
|
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
158
|
+
return HttpResponse(r.status, r.read().decode("utf-8", "replace"), {k.lower(): v for k, v in r.headers.items()})
|
|
159
|
+
except urllib.error.HTTPError as e:
|
|
160
|
+
try:
|
|
161
|
+
text = e.read().decode("utf-8", "replace")
|
|
162
|
+
except Exception:
|
|
163
|
+
text = ""
|
|
164
|
+
return HttpResponse(e.code, text, {k.lower(): v for k, v in (e.headers or {}).items()})
|
|
165
|
+
except (socket.timeout, TimeoutError):
|
|
166
|
+
raise TransportTimeout("timed out after %.0f s" % timeout) from None
|
|
167
|
+
except urllib.error.URLError as e:
|
|
168
|
+
if isinstance(e.reason, (socket.timeout, TimeoutError)):
|
|
169
|
+
raise TransportTimeout("timed out after %.0f s" % timeout) from None
|
|
170
|
+
raise TransportError("connection error: %s" % type(e.reason).__name__) from None
|
|
171
|
+
except OSError as e:
|
|
172
|
+
raise TransportError("connection error: %s" % type(e).__name__) from None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# ----------------------------------------------------------------------------- rate limiter
|
|
176
|
+
class RateLimiter:
|
|
177
|
+
"""Spaces calls at least 60 / requests_per_minute seconds apart (thread-safe, reservation style). None or 0 = unlimited."""
|
|
178
|
+
|
|
179
|
+
def __init__(self, requests_per_minute: Optional[float] = None, clock: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep):
|
|
180
|
+
self.interval = 60.0 / requests_per_minute if requests_per_minute else 0.0
|
|
181
|
+
self._clock, self._sleep = clock, sleep
|
|
182
|
+
self._next = 0.0
|
|
183
|
+
self._lock = threading.Lock()
|
|
184
|
+
|
|
185
|
+
def acquire(self) -> float:
|
|
186
|
+
"""Block until a call may start; returns the seconds waited."""
|
|
187
|
+
if not self.interval:
|
|
188
|
+
return 0.0
|
|
189
|
+
with self._lock:
|
|
190
|
+
now = self._clock()
|
|
191
|
+
start = max(now, self._next)
|
|
192
|
+
self._next = start + self.interval
|
|
193
|
+
wait = start - now
|
|
194
|
+
if wait > 0:
|
|
195
|
+
self._sleep(wait)
|
|
196
|
+
return max(wait, 0.0)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# ----------------------------------------------------------------------------- request / result
|
|
200
|
+
@dataclass
|
|
201
|
+
class LLMRequest:
|
|
202
|
+
provider: str # "ollama" | "groq" | "gemini"
|
|
203
|
+
model: str
|
|
204
|
+
prompt: str
|
|
205
|
+
system: Optional[str] = None
|
|
206
|
+
temperature: float = 0.0
|
|
207
|
+
seed: Optional[int] = None
|
|
208
|
+
top_p: float = 1.0
|
|
209
|
+
max_tokens: int = 2048
|
|
210
|
+
extra: Dict[str, Any] = field(default_factory=dict) # provider options, e.g. {"think": True} (Ollama), {"reasoning_effort": "low"} (Groq)
|
|
211
|
+
# labels, not part of the cache key
|
|
212
|
+
run_id: Optional[str] = None
|
|
213
|
+
item_id: Optional[str] = None
|
|
214
|
+
strategy: Optional[str] = None
|
|
215
|
+
round: Optional[int] = None
|
|
216
|
+
agent_id: Optional[str] = None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass
|
|
220
|
+
class Parsed:
|
|
221
|
+
text: str
|
|
222
|
+
prompt_tokens: Optional[int] = None
|
|
223
|
+
completion_tokens: Optional[int] = None
|
|
224
|
+
reasoning_tokens: Optional[int] = None
|
|
225
|
+
reasoning_chars: Optional[int] = None
|
|
226
|
+
finish_reason: Optional[str] = None
|
|
227
|
+
usage_raw: Optional[dict] = None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@dataclass
|
|
231
|
+
class LLMResult:
|
|
232
|
+
status: str # ok | timeout | error
|
|
233
|
+
text: Optional[str]
|
|
234
|
+
record: dict # the log line of the final attempt
|
|
235
|
+
cached: bool = False
|
|
236
|
+
error: Optional[str] = None
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def ok(self) -> bool:
|
|
240
|
+
return self.status == "ok"
|
|
241
|
+
|
|
242
|
+
def raise_for_status(self) -> None:
|
|
243
|
+
if not self.ok:
|
|
244
|
+
raise ClientError("call failed: status=%s error=%s" % (self.status, self.error))
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class ParseError(Exception):
|
|
248
|
+
pass
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def cache_key(req: LLMRequest) -> str:
|
|
252
|
+
"""sha256 over the JSON list [provider, model, prompt, temperature, seed, top_p, max_tokens] (+ system, extra only when set)."""
|
|
253
|
+
parts: List[Any] = [req.provider, req.model, req.prompt, req.temperature, req.seed, req.top_p, req.max_tokens]
|
|
254
|
+
if req.system:
|
|
255
|
+
parts.append({"system": req.system})
|
|
256
|
+
if req.extra:
|
|
257
|
+
parts.append({"extra": req.extra})
|
|
258
|
+
return hashlib.sha256(json.dumps(parts, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest()
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
_THINK_CLOSED = re.compile(r"<think>(.*?)</think>", re.S)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def think_chars(text: str) -> Optional[int]:
|
|
265
|
+
"""Characters inside <think> blocks of a response (an unclosed trailing block counts to the end); None when the text has no <think> tag."""
|
|
266
|
+
if not text or "<think>" not in text:
|
|
267
|
+
return None
|
|
268
|
+
total = sum(len(m) for m in _THINK_CLOSED.findall(text))
|
|
269
|
+
rest = _THINK_CLOSED.sub("", text)
|
|
270
|
+
if "<think>" in rest:
|
|
271
|
+
total += len(rest.split("<think>", 1)[1])
|
|
272
|
+
return total
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _int(x: Any) -> Optional[int]:
|
|
276
|
+
return x if isinstance(x, int) and not isinstance(x, bool) else None
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
# ----------------------------------------------------------------------------- providers
|
|
280
|
+
class Provider:
|
|
281
|
+
name = ""
|
|
282
|
+
key_env: Optional[str] = None
|
|
283
|
+
|
|
284
|
+
def __init__(self, base_url: str, user_agent: str = DEFAULT_USER_AGENT):
|
|
285
|
+
self.base_url = base_url.rstrip("/")
|
|
286
|
+
self.user_agent = user_agent
|
|
287
|
+
|
|
288
|
+
def build(self, req: LLMRequest, api_key: Optional[str]) -> Tuple[str, str, Dict[str, str], bytes]:
|
|
289
|
+
raise NotImplementedError
|
|
290
|
+
|
|
291
|
+
def parse(self, data: dict) -> Parsed:
|
|
292
|
+
raise NotImplementedError
|
|
293
|
+
|
|
294
|
+
@staticmethod
|
|
295
|
+
def _messages(req: LLMRequest) -> List[dict]:
|
|
296
|
+
msgs = []
|
|
297
|
+
if req.system:
|
|
298
|
+
msgs.append({"role": "system", "content": req.system})
|
|
299
|
+
msgs.append({"role": "user", "content": req.prompt})
|
|
300
|
+
return msgs
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
class OllamaProvider(Provider):
|
|
304
|
+
name = "ollama"
|
|
305
|
+
|
|
306
|
+
def build(self, req, api_key):
|
|
307
|
+
opts: Dict[str, Any] = {"temperature": req.temperature, "top_p": req.top_p, "num_predict": req.max_tokens}
|
|
308
|
+
if req.seed is not None:
|
|
309
|
+
opts["seed"] = req.seed
|
|
310
|
+
body: Dict[str, Any] = {"model": req.model, "messages": self._messages(req), "stream": False}
|
|
311
|
+
extra = dict(req.extra)
|
|
312
|
+
opts.update(extra.pop("options", {}) or {})
|
|
313
|
+
body.update(extra) # e.g. think, keep_alive
|
|
314
|
+
body["options"] = opts
|
|
315
|
+
return "POST", self.base_url + "/api/chat", {"Content-Type": "application/json", "User-Agent": self.user_agent}, json.dumps(body).encode("utf-8")
|
|
316
|
+
|
|
317
|
+
def parse(self, data):
|
|
318
|
+
msg = data.get("message")
|
|
319
|
+
if not isinstance(msg, dict) or "content" not in msg:
|
|
320
|
+
raise ParseError("no message.content in the Ollama response")
|
|
321
|
+
text = msg.get("content") or ""
|
|
322
|
+
thinking = msg.get("thinking")
|
|
323
|
+
rc = len(thinking) if isinstance(thinking, str) else think_chars(text)
|
|
324
|
+
usage = {k: data[k] for k in ("prompt_eval_count", "eval_count", "total_duration", "load_duration", "prompt_eval_duration", "eval_duration") if k in data}
|
|
325
|
+
return Parsed(text=text, prompt_tokens=_int(data.get("prompt_eval_count")), completion_tokens=_int(data.get("eval_count")),
|
|
326
|
+
reasoning_tokens=None, reasoning_chars=rc, finish_reason=data.get("done_reason"), usage_raw=usage or None)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class GroqProvider(Provider):
|
|
330
|
+
name = "groq"
|
|
331
|
+
key_env = "GROQ_API_KEY"
|
|
332
|
+
|
|
333
|
+
def build(self, req, api_key):
|
|
334
|
+
body: Dict[str, Any] = {"model": req.model, "messages": self._messages(req), "temperature": req.temperature, "top_p": req.top_p,
|
|
335
|
+
"max_completion_tokens": req.max_tokens}
|
|
336
|
+
if req.seed is not None:
|
|
337
|
+
body["seed"] = req.seed
|
|
338
|
+
body.update(req.extra)
|
|
339
|
+
headers = {"Content-Type": "application/json", "Authorization": "Bearer " + str(api_key), "User-Agent": self.user_agent}
|
|
340
|
+
return "POST", self.base_url + "/chat/completions", headers, json.dumps(body).encode("utf-8")
|
|
341
|
+
|
|
342
|
+
def parse(self, data):
|
|
343
|
+
try:
|
|
344
|
+
choice = data["choices"][0]
|
|
345
|
+
msg = choice["message"]
|
|
346
|
+
except (KeyError, IndexError, TypeError):
|
|
347
|
+
raise ParseError("no choices[0].message in the Groq response") from None
|
|
348
|
+
text = msg.get("content") or ""
|
|
349
|
+
reasoning = msg.get("reasoning")
|
|
350
|
+
rc = len(reasoning) if isinstance(reasoning, str) else think_chars(text)
|
|
351
|
+
usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
|
|
352
|
+
details = usage.get("completion_tokens_details") if isinstance(usage.get("completion_tokens_details"), dict) else {}
|
|
353
|
+
return Parsed(text=text, prompt_tokens=_int(usage.get("prompt_tokens")), completion_tokens=_int(usage.get("completion_tokens")),
|
|
354
|
+
reasoning_tokens=_int(details.get("reasoning_tokens")), reasoning_chars=rc, finish_reason=choice.get("finish_reason"),
|
|
355
|
+
usage_raw=usage or None)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class GeminiProvider(Provider):
|
|
359
|
+
name = "gemini"
|
|
360
|
+
key_env = "GEMINI_API_KEY"
|
|
361
|
+
|
|
362
|
+
def build(self, req, api_key):
|
|
363
|
+
gen: Dict[str, Any] = {"temperature": req.temperature, "topP": req.top_p, "maxOutputTokens": req.max_tokens}
|
|
364
|
+
if req.seed is not None:
|
|
365
|
+
gen["seed"] = req.seed
|
|
366
|
+
extra = dict(req.extra)
|
|
367
|
+
gen.update(extra.pop("generationConfig", {}) or {})
|
|
368
|
+
body: Dict[str, Any] = {"contents": [{"role": "user", "parts": [{"text": req.prompt}]}], "generationConfig": gen}
|
|
369
|
+
if req.system:
|
|
370
|
+
body["systemInstruction"] = {"parts": [{"text": req.system}]}
|
|
371
|
+
body.update(extra)
|
|
372
|
+
headers = {"Content-Type": "application/json", "x-goog-api-key": str(api_key), "User-Agent": self.user_agent}
|
|
373
|
+
return "POST", "%s/models/%s:generateContent" % (self.base_url, req.model), headers, json.dumps(body).encode("utf-8")
|
|
374
|
+
|
|
375
|
+
def parse(self, data):
|
|
376
|
+
cands = data.get("candidates")
|
|
377
|
+
if not cands:
|
|
378
|
+
fb = data.get("promptFeedback") or {}
|
|
379
|
+
raise ParseError("no candidates (blockReason=%s)" % fb.get("blockReason"))
|
|
380
|
+
cand = cands[0]
|
|
381
|
+
parts = ((cand.get("content") or {}).get("parts")) or []
|
|
382
|
+
text = "".join(p.get("text", "") for p in parts if isinstance(p, dict) and not p.get("thought"))
|
|
383
|
+
thought = [p.get("text", "") for p in parts if isinstance(p, dict) and p.get("thought")]
|
|
384
|
+
um = data.get("usageMetadata") if isinstance(data.get("usageMetadata"), dict) else {}
|
|
385
|
+
return Parsed(text=text, prompt_tokens=_int(um.get("promptTokenCount")), completion_tokens=_int(um.get("candidatesTokenCount")),
|
|
386
|
+
reasoning_tokens=_int(um.get("thoughtsTokenCount")), reasoning_chars=sum(len(t) for t in thought) if thought else None,
|
|
387
|
+
finish_reason=cand.get("finishReason"), usage_raw=um or None)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
# ----------------------------------------------------------------------------- the client
|
|
391
|
+
def _utc_iso(ts: float) -> str:
|
|
392
|
+
return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat(timespec="milliseconds")
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
class LLMClient:
|
|
396
|
+
def __init__(self, log_dir: str, run_id: Optional[str] = None, transport: Optional[Transport] = None,
|
|
397
|
+
rate_limits: Optional[Dict[str, Optional[float]]] = None, max_retries: int = 4, backoff_base: float = 1.0, backoff_max: float = 60.0,
|
|
398
|
+
timeout: float = DEFAULT_TIMEOUT, ollama_url: str = "http://localhost:11434", groq_url: str = "https://api.groq.com/openai/v1",
|
|
399
|
+
gemini_url: str = "https://generativelanguage.googleapis.com/v1beta", user_agent: str = DEFAULT_USER_AGENT,
|
|
400
|
+
environ: Optional[Dict[str, str]] = None, extra_secrets=(), sleep: Callable[[float], None] = time.sleep,
|
|
401
|
+
monotonic: Callable[[], float] = time.monotonic, wall_clock: Callable[[], float] = time.time,
|
|
402
|
+
rng: Optional[random.Random] = None, log_name: str = "calls.jsonl", cache_name: str = "cache.jsonl"):
|
|
403
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
404
|
+
self.log_path = os.path.join(log_dir, log_name)
|
|
405
|
+
self.cache_path = os.path.join(log_dir, cache_name)
|
|
406
|
+
self.run_id = run_id or (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:6])
|
|
407
|
+
self.transport = transport or UrllibTransport()
|
|
408
|
+
self.max_retries, self.backoff_base, self.backoff_max, self.timeout = max_retries, backoff_base, backoff_max, timeout
|
|
409
|
+
self._environ = os.environ if environ is None else environ
|
|
410
|
+
self._sleep, self._wall = sleep, wall_clock
|
|
411
|
+
self._mono = monotonic
|
|
412
|
+
self._rng = rng or random.Random()
|
|
413
|
+
self.providers: Dict[str, Provider] = {"ollama": OllamaProvider(ollama_url, user_agent), "groq": GroqProvider(groq_url, user_agent),
|
|
414
|
+
"gemini": GeminiProvider(gemini_url, user_agent)}
|
|
415
|
+
limits = dict(DEFAULT_RPM)
|
|
416
|
+
limits.update(rate_limits or {})
|
|
417
|
+
self.limiters = {p: RateLimiter(limits.get(p), clock=monotonic, sleep=sleep) for p in self.providers}
|
|
418
|
+
self.redact = Redactor(extra_secrets)
|
|
419
|
+
for var in SECRET_ENV_VARS:
|
|
420
|
+
self.redact.add(self._environ.get(var))
|
|
421
|
+
self._io_lock = threading.Lock()
|
|
422
|
+
self._key_locks: Dict[str, threading.Lock] = {}
|
|
423
|
+
self._digests: Dict[str, Optional[str]] = {}
|
|
424
|
+
self._cache: Dict[str, dict] = {}
|
|
425
|
+
self._load_cache()
|
|
426
|
+
|
|
427
|
+
def __repr__(self) -> str:
|
|
428
|
+
return "LLMClient(run_id=%r, log=%r)" % (self.run_id, self.log_path)
|
|
429
|
+
|
|
430
|
+
# --- persistence
|
|
431
|
+
def _load_cache(self) -> None:
|
|
432
|
+
if not os.path.isfile(self.cache_path):
|
|
433
|
+
return
|
|
434
|
+
with open(self.cache_path, encoding="utf-8") as fh:
|
|
435
|
+
for line in fh:
|
|
436
|
+
line = line.strip()
|
|
437
|
+
if not line:
|
|
438
|
+
continue
|
|
439
|
+
try:
|
|
440
|
+
rec = json.loads(line)
|
|
441
|
+
except ValueError:
|
|
442
|
+
continue # a half-written last line after a crash
|
|
443
|
+
if isinstance(rec, dict) and rec.get("cache_key") and rec.get("status") == "ok":
|
|
444
|
+
self._cache.setdefault(rec["cache_key"], rec)
|
|
445
|
+
|
|
446
|
+
def _append(self, path: str, rec: dict) -> None:
|
|
447
|
+
line = self.redact(json.dumps(rec, ensure_ascii=False))
|
|
448
|
+
with self._io_lock:
|
|
449
|
+
with open(path, "a", encoding="utf-8", newline="\n") as fh:
|
|
450
|
+
fh.write(line + "\n")
|
|
451
|
+
fh.flush()
|
|
452
|
+
|
|
453
|
+
@property
|
|
454
|
+
def cache_size(self) -> int:
|
|
455
|
+
return len(self._cache)
|
|
456
|
+
|
|
457
|
+
# --- helpers
|
|
458
|
+
def _api_key(self, provider: Provider) -> Optional[str]:
|
|
459
|
+
if not provider.key_env:
|
|
460
|
+
return None
|
|
461
|
+
val = self._environ.get(provider.key_env)
|
|
462
|
+
if not val:
|
|
463
|
+
raise MissingKeyError("environment variable %s is not set" % provider.key_env)
|
|
464
|
+
self.redact.add(val)
|
|
465
|
+
return val
|
|
466
|
+
|
|
467
|
+
def model_digest(self, model: str) -> Optional[str]:
|
|
468
|
+
"""Digest of a local Ollama model from /api/tags (None when the daemon or the model is not found). Fetched once per model."""
|
|
469
|
+
with self._io_lock:
|
|
470
|
+
if model in self._digests:
|
|
471
|
+
return self._digests[model]
|
|
472
|
+
digest = None
|
|
473
|
+
try:
|
|
474
|
+
r = self.transport.request("GET", self.providers["ollama"].base_url + "/api/tags", {"User-Agent": DEFAULT_USER_AGENT}, None, min(self.timeout, 30.0))
|
|
475
|
+
if r.status == 200:
|
|
476
|
+
models = json.loads(r.body).get("models", [])
|
|
477
|
+
wanted = {model, model if ":" in model else model + ":latest"}
|
|
478
|
+
for m in models:
|
|
479
|
+
if m.get("name") in wanted or m.get("model") in wanted:
|
|
480
|
+
digest = m.get("digest")
|
|
481
|
+
break
|
|
482
|
+
except Exception:
|
|
483
|
+
digest = None
|
|
484
|
+
with self._io_lock:
|
|
485
|
+
self._digests[model] = digest
|
|
486
|
+
return digest
|
|
487
|
+
|
|
488
|
+
def _line(self, req: LLMRequest, key: str, attempt: int, status: str, digest: Optional[str], parsed: Optional[Parsed] = None,
|
|
489
|
+
latency: Optional[float] = None, http_status: Optional[int] = None, error: Optional[str] = None, cached: bool = False) -> dict:
|
|
490
|
+
p = parsed
|
|
491
|
+
return {
|
|
492
|
+
"run_id": req.run_id or self.run_id, "item_id": req.item_id, "strategy": req.strategy, "round": req.round, "agent_id": req.agent_id,
|
|
493
|
+
"timestamp": _utc_iso(self._wall()), "provider": req.provider, "model": req.model, "model_digest": digest,
|
|
494
|
+
"temperature": req.temperature, "seed": req.seed, "top_p": req.top_p, "max_tokens": req.max_tokens,
|
|
495
|
+
"prompt_tokens": p.prompt_tokens if p else None, "completion_tokens": p.completion_tokens if p else None,
|
|
496
|
+
"reasoning_tokens": p.reasoning_tokens if p else None, "reasoning_chars": p.reasoning_chars if p else None,
|
|
497
|
+
"latency_s": latency, "text": p.text if p else None, "status": status, "attempt": attempt, "cached": cached,
|
|
498
|
+
"http_status": http_status, "finish_reason": p.finish_reason if p else None, "error": error,
|
|
499
|
+
"usage_raw": p.usage_raw if p else None, "cache_key": key,
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
def _delay(self, attempt: int, resp: Optional[HttpResponse]) -> float:
|
|
503
|
+
if resp is not None:
|
|
504
|
+
ra = resp.headers.get("retry-after")
|
|
505
|
+
if ra:
|
|
506
|
+
try:
|
|
507
|
+
return min(self.backoff_max, max(0.0, float(ra)))
|
|
508
|
+
except ValueError:
|
|
509
|
+
pass
|
|
510
|
+
return min(self.backoff_max, self.backoff_base * (2 ** (attempt - 1)) * (1.0 + 0.25 * self._rng.random()))
|
|
511
|
+
|
|
512
|
+
# --- the call
|
|
513
|
+
def call(self, req: LLMRequest) -> LLMResult:
|
|
514
|
+
if req.provider not in self.providers:
|
|
515
|
+
raise ClientError("unknown provider %r (expected one of %s)" % (req.provider, sorted(self.providers)))
|
|
516
|
+
key = cache_key(req)
|
|
517
|
+
with self._io_lock:
|
|
518
|
+
lock = self._key_locks.setdefault(key, threading.Lock())
|
|
519
|
+
with lock:
|
|
520
|
+
hit = self._cache.get(key)
|
|
521
|
+
if hit is not None:
|
|
522
|
+
p = Parsed(text=hit.get("text") or "", prompt_tokens=hit.get("prompt_tokens"), completion_tokens=hit.get("completion_tokens"),
|
|
523
|
+
reasoning_tokens=hit.get("reasoning_tokens"), reasoning_chars=hit.get("reasoning_chars"), finish_reason=hit.get("finish_reason"),
|
|
524
|
+
usage_raw=hit.get("usage_raw"))
|
|
525
|
+
rec = self._line(req, key, 0, "cache_hit", hit.get("model_digest"), p, None, hit.get("http_status"), None, True)
|
|
526
|
+
self._append(self.log_path, rec)
|
|
527
|
+
return LLMResult("ok", p.text, rec, cached=True)
|
|
528
|
+
provider = self.providers[req.provider]
|
|
529
|
+
api_key = self._api_key(provider)
|
|
530
|
+
digest = self.model_digest(req.model) if req.provider == "ollama" else None
|
|
531
|
+
method, url, headers, body = provider.build(req, api_key)
|
|
532
|
+
last: Optional[LLMResult] = None
|
|
533
|
+
for attempt in range(1, self.max_retries + 2):
|
|
534
|
+
self.limiters[req.provider].acquire()
|
|
535
|
+
t0 = self._mono()
|
|
536
|
+
resp: Optional[HttpResponse] = None
|
|
537
|
+
status, err, retry, parsed = "error", None, False, None
|
|
538
|
+
try:
|
|
539
|
+
resp = self.transport.request(method, url, headers, body, self.timeout)
|
|
540
|
+
except TransportTimeout as e:
|
|
541
|
+
status, err, retry = "timeout", self.redact(e), True
|
|
542
|
+
except TransportError as e:
|
|
543
|
+
status, err, retry = "error", self.redact(e), True
|
|
544
|
+
except Exception as e: # a misbehaving transport must not leak its message (it may contain the URL or headers)
|
|
545
|
+
status, err, retry = "error", self.redact("%s: %s" % (type(e).__name__, e)), False
|
|
546
|
+
latency = self._mono() - t0
|
|
547
|
+
http_status = resp.status if resp is not None else None
|
|
548
|
+
if resp is not None:
|
|
549
|
+
if resp.status == 200:
|
|
550
|
+
try:
|
|
551
|
+
parsed = provider.parse(json.loads(resp.body))
|
|
552
|
+
status = "ok"
|
|
553
|
+
except (ParseError, ValueError, TypeError, AttributeError) as e:
|
|
554
|
+
status, err = "error", self.redact("unparseable response: %s" % e)
|
|
555
|
+
else:
|
|
556
|
+
status, err = "error", self.redact("HTTP %d: %s" % (resp.status, resp.body[:300].replace("\n", " ")))
|
|
557
|
+
retry = resp.status in RETRY_STATUSES
|
|
558
|
+
rec = self._line(req, key, attempt, status, digest, parsed, round(latency, 6), http_status, err)
|
|
559
|
+
self._append(self.log_path, rec)
|
|
560
|
+
last = LLMResult(status, parsed.text if parsed else None, rec, False, err)
|
|
561
|
+
if status == "ok":
|
|
562
|
+
self._append(self.cache_path, rec)
|
|
563
|
+
self._cache[key] = json.loads(self.redact(json.dumps(rec, ensure_ascii=False)))
|
|
564
|
+
return last
|
|
565
|
+
if not retry or attempt > self.max_retries:
|
|
566
|
+
return last
|
|
567
|
+
self._sleep(self._delay(attempt, resp))
|
|
568
|
+
return last # pragma: no cover
|