touchstone-llm 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- touchstone_llm/__init__.py +3 -0
- touchstone_llm/backends/__init__.py +112 -0
- touchstone_llm/backends/base.py +207 -0
- touchstone_llm/backends/errors.py +41 -0
- touchstone_llm/backends/gemini.py +73 -0
- touchstone_llm/backends/mock.py +38 -0
- touchstone_llm/backends/openai_compat.py +57 -0
- touchstone_llm/backends/ratelimit.py +79 -0
- touchstone_llm/cache.py +59 -0
- touchstone_llm/cli.py +337 -0
- touchstone_llm/config.py +83 -0
- touchstone_llm/drafts.py +26 -0
- touchstone_llm/env.py +146 -0
- touchstone_llm/errors.py +11 -0
- touchstone_llm/py.typed +0 -0
- touchstone_llm/report/__init__.py +84 -0
- touchstone_llm/report/templates/report.html.j2 +152 -0
- touchstone_llm/report/templates/report.md.j2 +56 -0
- touchstone_llm/reproduce.py +63 -0
- touchstone_llm/results.py +138 -0
- touchstone_llm/review.py +243 -0
- touchstone_llm/runner.py +152 -0
- touchstone_llm/scoring/__init__.py +1 -0
- touchstone_llm/scoring/chrf.py +54 -0
- touchstone_llm/scoring/mcq.py +107 -0
- touchstone_llm/scoring/normalize.py +37 -0
- touchstone_llm/scoring/numeric.py +37 -0
- touchstone_llm/scoring/rules.py +176 -0
- touchstone_llm/server/__init__.py +1 -0
- touchstone_llm/server/app.py +264 -0
- touchstone_llm/server/jobs.py +114 -0
- touchstone_llm/server/static/assets/fira-code-latin-400-normal-C-QZfXAs.woff +0 -0
- touchstone_llm/server/static/assets/fira-code-latin-400-normal-DGosTW8U.woff2 +0 -0
- touchstone_llm/server/static/assets/fira-code-latin-500-normal-B6gioGu8.woff2 +0 -0
- touchstone_llm/server/static/assets/fira-code-latin-500-normal-aMLC2AMm.woff +0 -0
- touchstone_llm/server/static/assets/index-Dgv2BMCR.css +2 -0
- touchstone_llm/server/static/assets/index-Pvf3wJWw.js +11 -0
- touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-400-normal-BlDd6KY1.woff2 +0 -0
- touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-400-normal-CdOJmQ4E.woff +0 -0
- touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-500-normal-DO6LFp12.woff +0 -0
- touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-500-normal-DVtwyDnM.woff2 +0 -0
- touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-600-normal-B5tDaqnD.woff +0 -0
- touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-600-normal-BeyktMLo.woff2 +0 -0
- touchstone_llm/server/static/assets/noto-sans-latin-400-normal-BTkUljjl.woff2 +0 -0
- touchstone_llm/server/static/assets/noto-sans-latin-400-normal-DPpKfCgK.woff +0 -0
- touchstone_llm/server/static/assets/noto-sans-latin-500-normal-CDPngY5C.woff +0 -0
- touchstone_llm/server/static/assets/noto-sans-latin-500-normal-DCA2Z9Cj.woff2 +0 -0
- touchstone_llm/server/static/assets/noto-sans-latin-600-normal-DVJMj_t0.woff +0 -0
- touchstone_llm/server/static/assets/noto-sans-latin-600-normal-Dkgu5AE5.woff2 +0 -0
- touchstone_llm/server/static/index.html +22 -0
- touchstone_llm/tasks/__init__.py +61 -0
- touchstone_llm/tasks/base.py +128 -0
- touchstone_llm/tasks/data/drafts/codemixed_instructions.jsonl +25 -0
- touchstone_llm/tasks/data/drafts/cultural_context.jsonl +25 -0
- touchstone_llm/tasks/data/drafts/hindi_comprehension.jsonl +25 -0
- touchstone_llm/tasks/data/drafts/hindi_reasoning.jsonl +25 -0
- touchstone_llm/tasks/data/drafts/hinglish_understanding.jsonl +25 -0
- touchstone_llm/tasks/data/drafts/translation_hi_en.jsonl +25 -0
- touchstone_llm/tasks/instructions.py +38 -0
- touchstone_llm/tasks/mcq.py +93 -0
- touchstone_llm/tasks/numeric.py +52 -0
- touchstone_llm/tasks/translation.py +51 -0
- touchstone_llm-0.1.0.dist-info/METADATA +319 -0
- touchstone_llm-0.1.0.dist-info/RECORD +67 -0
- touchstone_llm-0.1.0.dist-info/WHEEL +4 -0
- touchstone_llm-0.1.0.dist-info/entry_points.txt +2 -0
- touchstone_llm-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Model backends and the provider registry.
|
|
2
|
+
|
|
3
|
+
Built-in providers: ``groq``, ``gemini``, ``openrouter``, ``ollama`` and the
|
|
4
|
+
offline ``mock``. Third-party packages can add more through the
|
|
5
|
+
``touchstone_llm.backends`` entry point group; each entry point must load a
|
|
6
|
+
callable that takes keyword overrides and returns a :class:`Backend`.
|
|
7
|
+
|
|
8
|
+
Default request rates are cautious guesses for free tiers. Providers change
|
|
9
|
+
their limits, so pass ``requests_per_minute`` to override them.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from importlib.metadata import entry_points
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from touchstone_llm.backends.base import Backend, Completion
|
|
20
|
+
from touchstone_llm.backends.gemini import GeminiBackend
|
|
21
|
+
from touchstone_llm.backends.mock import MockBackend
|
|
22
|
+
from touchstone_llm.backends.openai_compat import OpenAICompatBackend
|
|
23
|
+
from touchstone_llm.env import get_api_key
|
|
24
|
+
from touchstone_llm.errors import TouchstoneError
|
|
25
|
+
|
|
26
|
+
__all__ = ["Backend", "Completion", "available_backends", "get_backend"]
|
|
27
|
+
|
|
28
|
+
BackendFactory = Callable[..., Backend]
|
|
29
|
+
|
|
30
|
+
DEFAULT_OLLAMA_URL = "http://localhost:11434"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _groq(**overrides: Any) -> Backend:
|
|
34
|
+
options: dict[str, Any] = {"requests_per_minute": 30, **overrides}
|
|
35
|
+
return OpenAICompatBackend(
|
|
36
|
+
name="groq",
|
|
37
|
+
base_url="https://api.groq.com/openai/v1",
|
|
38
|
+
api_key=get_api_key("groq"),
|
|
39
|
+
**options,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _openrouter(**overrides: Any) -> Backend:
|
|
44
|
+
options: dict[str, Any] = {"requests_per_minute": 20, **overrides}
|
|
45
|
+
return OpenAICompatBackend(
|
|
46
|
+
name="openrouter",
|
|
47
|
+
base_url="https://openrouter.ai/api/v1",
|
|
48
|
+
api_key=get_api_key("openrouter"),
|
|
49
|
+
extra_headers={"X-Title": "touchstone"},
|
|
50
|
+
**options,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _ollama(**overrides: Any) -> Backend:
|
|
55
|
+
base = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_URL).strip() or DEFAULT_OLLAMA_URL
|
|
56
|
+
options: dict[str, Any] = {"requests_per_minute": None, "timeout": 300.0, **overrides}
|
|
57
|
+
return OpenAICompatBackend(
|
|
58
|
+
name="ollama",
|
|
59
|
+
base_url=f"{base.rstrip('/')}/v1",
|
|
60
|
+
api_key=None,
|
|
61
|
+
connect_hint="Is Ollama running? Start it with `ollama serve`, then pull a small "
|
|
62
|
+
"model, e.g. `ollama pull <model>`.",
|
|
63
|
+
**options,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _gemini(**overrides: Any) -> Backend:
|
|
68
|
+
options: dict[str, Any] = {"requests_per_minute": 10, **overrides}
|
|
69
|
+
key = get_api_key("gemini")
|
|
70
|
+
assert key is not None # gemini always needs a key; get_api_key raises otherwise
|
|
71
|
+
return GeminiBackend(api_key=key, **options)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _mock(**_overrides: Any) -> Backend:
|
|
75
|
+
return MockBackend()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
_BUILTINS: dict[str, BackendFactory] = {
|
|
79
|
+
"groq": _groq,
|
|
80
|
+
"gemini": _gemini,
|
|
81
|
+
"openrouter": _openrouter,
|
|
82
|
+
"ollama": _ollama,
|
|
83
|
+
"mock": _mock,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _plugins() -> dict[str, BackendFactory]:
|
|
88
|
+
found: dict[str, BackendFactory] = {}
|
|
89
|
+
for ep in entry_points(group="touchstone_llm.backends"):
|
|
90
|
+
if ep.name not in _BUILTINS:
|
|
91
|
+
found[ep.name] = ep.load()
|
|
92
|
+
return found
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def available_backends() -> list[str]:
|
|
96
|
+
"""Names of all built-in and plugin providers, sorted."""
|
|
97
|
+
return sorted({*_BUILTINS, *(ep.name for ep in entry_points(group="touchstone_llm.backends"))})
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def get_backend(provider: str, **overrides: Any) -> Backend:
|
|
101
|
+
"""Create the backend for ``provider``.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
TouchstoneError: unknown provider (the message lists known ones).
|
|
105
|
+
MissingAPIKeyError: the provider needs a key that is not set.
|
|
106
|
+
|
|
107
|
+
"""
|
|
108
|
+
factory = _BUILTINS.get(provider) or _plugins().get(provider)
|
|
109
|
+
if factory is None:
|
|
110
|
+
known = ", ".join(available_backends())
|
|
111
|
+
raise TouchstoneError(f"Unknown provider '{provider}'. Available providers: {known}.")
|
|
112
|
+
return factory(**overrides)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Backend interface shared by every model provider.
|
|
2
|
+
|
|
3
|
+
To add a provider, subclass :class:`Backend` (or :class:`HTTPBackend`),
|
|
4
|
+
implement ``_generate`` and ``list_models``, and register a factory (see
|
|
5
|
+
``touchstone_llm.backends``).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
from pydantic import BaseModel
|
|
17
|
+
|
|
18
|
+
from touchstone_llm import __version__
|
|
19
|
+
from touchstone_llm.backends.errors import (
|
|
20
|
+
AuthError,
|
|
21
|
+
BackendError,
|
|
22
|
+
ModelNotFoundError,
|
|
23
|
+
RateLimitError,
|
|
24
|
+
TransientError,
|
|
25
|
+
)
|
|
26
|
+
from touchstone_llm.backends.ratelimit import RateLimiter, with_retries
|
|
27
|
+
from touchstone_llm.config import GenerationSettings
|
|
28
|
+
from touchstone_llm.env import PROVIDER_KEYS, redact_secrets
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"AuthError",
|
|
32
|
+
"Backend",
|
|
33
|
+
"BackendError",
|
|
34
|
+
"Completion",
|
|
35
|
+
"HTTPBackend",
|
|
36
|
+
"ModelNotFoundError",
|
|
37
|
+
"RateLimitError",
|
|
38
|
+
"TransientError",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
_MODEL_MISSING_HINTS = (
|
|
42
|
+
"model_not_found",
|
|
43
|
+
"does not exist",
|
|
44
|
+
"decommissioned",
|
|
45
|
+
"not a valid model",
|
|
46
|
+
"not found",
|
|
47
|
+
"no endpoints found",
|
|
48
|
+
"is not supported",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Completion(BaseModel):
|
|
53
|
+
"""A model's reply to one prompt."""
|
|
54
|
+
|
|
55
|
+
text: str
|
|
56
|
+
model_id: str | None = None
|
|
57
|
+
"""Exact model id or version the provider reports, if any."""
|
|
58
|
+
finish_reason: str | None = None
|
|
59
|
+
latency_ms: float = 0.0
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Backend(ABC):
|
|
63
|
+
"""Base class: adds rate limiting, retries and timing around ``_generate``."""
|
|
64
|
+
|
|
65
|
+
name: str
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self,
|
|
69
|
+
*,
|
|
70
|
+
requests_per_minute: float | None,
|
|
71
|
+
max_attempts: int = 5,
|
|
72
|
+
base_delay: float = 2.0,
|
|
73
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
74
|
+
clock: Callable[[], float] = time.monotonic,
|
|
75
|
+
) -> None:
|
|
76
|
+
"""Set up the rate limiter and retry policy."""
|
|
77
|
+
self._limiter = RateLimiter(requests_per_minute, clock=clock, sleep=sleep)
|
|
78
|
+
self._max_attempts = max_attempts
|
|
79
|
+
self._base_delay = base_delay
|
|
80
|
+
self._sleep = sleep
|
|
81
|
+
self._clock = clock
|
|
82
|
+
|
|
83
|
+
def generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
|
|
84
|
+
"""Send one prompt and return the reply, retrying temporary failures."""
|
|
85
|
+
|
|
86
|
+
def attempt() -> Completion:
|
|
87
|
+
self._limiter.acquire()
|
|
88
|
+
start = self._clock()
|
|
89
|
+
completion = self._generate(model_id, prompt, settings)
|
|
90
|
+
elapsed = (self._clock() - start) * 1000
|
|
91
|
+
return completion.model_copy(update={"latency_ms": max(elapsed, 0.0)})
|
|
92
|
+
|
|
93
|
+
return with_retries(
|
|
94
|
+
attempt,
|
|
95
|
+
max_attempts=self._max_attempts,
|
|
96
|
+
base_delay=self._base_delay,
|
|
97
|
+
sleep=self._sleep,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
@abstractmethod
|
|
101
|
+
def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
|
|
102
|
+
"""Make exactly one provider call."""
|
|
103
|
+
|
|
104
|
+
@abstractmethod
|
|
105
|
+
def list_models(self) -> list[str]:
|
|
106
|
+
"""Return model ids the provider currently offers, sorted."""
|
|
107
|
+
|
|
108
|
+
def close(self) -> None: # noqa: B027 (optional hook, default does nothing)
|
|
109
|
+
"""Release network resources."""
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class HTTPBackend(Backend):
|
|
113
|
+
"""A backend that talks to a JSON HTTP API with ``httpx``."""
|
|
114
|
+
|
|
115
|
+
def __init__(
|
|
116
|
+
self,
|
|
117
|
+
*,
|
|
118
|
+
name: str,
|
|
119
|
+
base_url: str,
|
|
120
|
+
api_key: str | None,
|
|
121
|
+
headers: dict[str, str] | None = None,
|
|
122
|
+
timeout: float = 60.0,
|
|
123
|
+
connect_hint: str | None = None,
|
|
124
|
+
**kwargs: Any,
|
|
125
|
+
) -> None:
|
|
126
|
+
"""Create the HTTP client. ``connect_hint`` is shown when the server is unreachable."""
|
|
127
|
+
super().__init__(**kwargs)
|
|
128
|
+
self.name = name
|
|
129
|
+
self.base_url = base_url.rstrip("/")
|
|
130
|
+
self._api_key = api_key
|
|
131
|
+
self._connect_hint = connect_hint
|
|
132
|
+
self._client = httpx.Client(
|
|
133
|
+
base_url=self.base_url,
|
|
134
|
+
headers={"User-Agent": f"touchstone/{__version__}", **(headers or {})},
|
|
135
|
+
timeout=timeout,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
def close(self) -> None:
|
|
139
|
+
"""Close the HTTP connection pool."""
|
|
140
|
+
self._client.close()
|
|
141
|
+
|
|
142
|
+
def _redact(self, text: str) -> str:
|
|
143
|
+
text = redact_secrets(text)
|
|
144
|
+
if self._api_key:
|
|
145
|
+
text = text.replace(self._api_key, "***")
|
|
146
|
+
return text
|
|
147
|
+
|
|
148
|
+
def _request(self, method: str, path: str, model_id: str = "", **kwargs: Any) -> Any:
|
|
149
|
+
"""Send a request and return parsed JSON, mapping failures to backend errors."""
|
|
150
|
+
try:
|
|
151
|
+
response = self._client.request(method, path, **kwargs)
|
|
152
|
+
except httpx.ConnectError as err:
|
|
153
|
+
hint = f"\n{self._connect_hint}" if self._connect_hint else ""
|
|
154
|
+
raise BackendError(
|
|
155
|
+
f"Could not connect to {self.name} at {self.base_url}.{hint}"
|
|
156
|
+
) from err
|
|
157
|
+
except httpx.TransportError as err:
|
|
158
|
+
raise TransientError(f"Network problem talking to {self.name}: {err}") from err
|
|
159
|
+
if response.is_success:
|
|
160
|
+
return response.json()
|
|
161
|
+
self._raise_for_status(response, model_id)
|
|
162
|
+
|
|
163
|
+
def _raise_for_status(self, response: httpx.Response, model_id: str) -> None:
|
|
164
|
+
status = response.status_code
|
|
165
|
+
detail = self._redact(_error_detail(response))
|
|
166
|
+
lowered = detail.lower()
|
|
167
|
+
if status in (401, 403) or "api key not valid" in lowered or "api_key_invalid" in lowered:
|
|
168
|
+
key = PROVIDER_KEYS.get(self.name)
|
|
169
|
+
where = f" Check {key.env_var} in your .env file." if key else ""
|
|
170
|
+
raise AuthError(f"{self.name} rejected the API key (HTTP {status}).{where}")
|
|
171
|
+
if status == 429:
|
|
172
|
+
raise RateLimitError(
|
|
173
|
+
f"{self.name} rate limit reached (HTTP 429). {detail}".strip(),
|
|
174
|
+
retry_after=_retry_after(response),
|
|
175
|
+
)
|
|
176
|
+
if status >= 500:
|
|
177
|
+
raise TransientError(f"{self.name} server error (HTTP {status}). {detail}".strip())
|
|
178
|
+
if status == 404 or (status == 400 and any(h in lowered for h in _MODEL_MISSING_HINTS)):
|
|
179
|
+
raise ModelNotFoundError(self.name, model_id, detail)
|
|
180
|
+
raise BackendError(f"{self.name} returned HTTP {status}. {detail}".strip())
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _error_detail(response: httpx.Response) -> str:
|
|
184
|
+
"""Best-effort error text from a provider's JSON error body."""
|
|
185
|
+
try:
|
|
186
|
+
data = response.json()
|
|
187
|
+
except ValueError:
|
|
188
|
+
return response.text[:300]
|
|
189
|
+
if isinstance(data, dict):
|
|
190
|
+
error = data.get("error")
|
|
191
|
+
if isinstance(error, dict):
|
|
192
|
+
parts = [str(error.get("message", "")), str(error.get("code", ""))]
|
|
193
|
+
parts += [
|
|
194
|
+
str(d.get("reason", "")) for d in error.get("details", []) if isinstance(d, dict)
|
|
195
|
+
]
|
|
196
|
+
return " ".join(p for p in parts if p and p != "None")
|
|
197
|
+
if isinstance(error, str):
|
|
198
|
+
return error
|
|
199
|
+
return ""
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _retry_after(response: httpx.Response) -> float | None:
|
|
203
|
+
value = response.headers.get("retry-after")
|
|
204
|
+
try:
|
|
205
|
+
return float(value) if value is not None else None
|
|
206
|
+
except ValueError:
|
|
207
|
+
return None
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Errors raised by model backends."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from touchstone_llm.errors import TouchstoneError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BackendError(TouchstoneError):
|
|
9
|
+
"""A model provider call failed in a way that retrying will not fix."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AuthError(BackendError):
|
|
13
|
+
"""The provider rejected the API key."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ModelNotFoundError(BackendError):
|
|
17
|
+
"""The model id is unknown, misspelled or retired."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, provider: str, model_id: str, detail: str = "") -> None:
|
|
20
|
+
"""Explain that the id may be retired and how to list current ids."""
|
|
21
|
+
msg = (
|
|
22
|
+
f"Model '{model_id}' is not available on {provider} "
|
|
23
|
+
f"(it may be retired or misspelled).\n"
|
|
24
|
+
f"Run `touchstone list-models {provider}` to see current model ids."
|
|
25
|
+
)
|
|
26
|
+
if detail:
|
|
27
|
+
msg += f"\nProvider said: {detail}"
|
|
28
|
+
super().__init__(msg)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RateLimitError(BackendError):
|
|
32
|
+
"""The provider asked us to slow down (HTTP 429). Retryable."""
|
|
33
|
+
|
|
34
|
+
def __init__(self, message: str, retry_after: float | None = None) -> None:
|
|
35
|
+
"""Store the provider's suggested wait in seconds, if it sent one."""
|
|
36
|
+
super().__init__(message)
|
|
37
|
+
self.retry_after = retry_after
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TransientError(BackendError):
|
|
41
|
+
"""A temporary failure (server error, timeout). Retryable."""
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Backend for the Google Gemini API (REST, no SDK).
|
|
2
|
+
|
|
3
|
+
The key is sent in the ``x-goog-api-key`` header, never in the URL, so it
|
|
4
|
+
does not end up in logs or error messages that print URLs.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from touchstone_llm.backends.base import Completion, HTTPBackend
|
|
12
|
+
from touchstone_llm.config import GenerationSettings
|
|
13
|
+
|
|
14
|
+
GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class GeminiBackend(HTTPBackend):
|
|
18
|
+
"""Calls ``models/{id}:generateContent`` on the Gemini API."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, *, api_key: str, base_url: str = GEMINI_BASE_URL, **kwargs: Any) -> None:
|
|
21
|
+
"""Create the backend with the key in a request header."""
|
|
22
|
+
super().__init__(
|
|
23
|
+
name="gemini",
|
|
24
|
+
base_url=base_url,
|
|
25
|
+
api_key=api_key,
|
|
26
|
+
headers={"x-goog-api-key": api_key},
|
|
27
|
+
**kwargs,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
|
|
31
|
+
model_id = model_id.removeprefix("models/")
|
|
32
|
+
data = self._request(
|
|
33
|
+
"POST",
|
|
34
|
+
f"/models/{model_id}:generateContent",
|
|
35
|
+
model_id=model_id,
|
|
36
|
+
json={
|
|
37
|
+
"contents": [{"role": "user", "parts": [{"text": prompt}]}],
|
|
38
|
+
"generationConfig": {
|
|
39
|
+
"temperature": settings.temperature,
|
|
40
|
+
"topP": settings.top_p,
|
|
41
|
+
"maxOutputTokens": settings.max_tokens,
|
|
42
|
+
"seed": settings.seed,
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
)
|
|
46
|
+
candidates = data.get("candidates") or []
|
|
47
|
+
if not candidates:
|
|
48
|
+
block = (data.get("promptFeedback") or {}).get("blockReason")
|
|
49
|
+
return Completion(text="", model_id=data.get("modelVersion"), finish_reason=block)
|
|
50
|
+
first = candidates[0]
|
|
51
|
+
parts = (first.get("content") or {}).get("parts") or []
|
|
52
|
+
text = "".join(p.get("text", "") for p in parts if not p.get("thought"))
|
|
53
|
+
return Completion(
|
|
54
|
+
text=text,
|
|
55
|
+
model_id=data.get("modelVersion"),
|
|
56
|
+
finish_reason=first.get("finishReason"),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def list_models(self) -> list[str]:
|
|
60
|
+
"""Return ids of models that support ``generateContent``, following pagination."""
|
|
61
|
+
ids: list[str] = []
|
|
62
|
+
token: str | None = None
|
|
63
|
+
while True:
|
|
64
|
+
params: dict[str, str | int] = {"pageSize": 1000}
|
|
65
|
+
if token:
|
|
66
|
+
params["pageToken"] = token
|
|
67
|
+
data = self._request("GET", "/models", params=params)
|
|
68
|
+
for model in data.get("models", []):
|
|
69
|
+
if "generateContent" in model.get("supportedGenerationMethods", []):
|
|
70
|
+
ids.append(str(model["name"]).removeprefix("models/"))
|
|
71
|
+
token = data.get("nextPageToken")
|
|
72
|
+
if not token:
|
|
73
|
+
return sorted(ids)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Offline backend for tests and trying touchstone without any API key.
|
|
2
|
+
|
|
3
|
+
Models:
|
|
4
|
+
``mock/echo``: replies with the prompt itself.
|
|
5
|
+
``mock/fixed``: always replies ``"A"``.
|
|
6
|
+
Either can be given scripted ``replies`` keyed by the exact prompt.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from touchstone_llm.backends.base import Backend, Completion, ModelNotFoundError
|
|
12
|
+
from touchstone_llm.config import GenerationSettings
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class MockBackend(Backend):
|
|
16
|
+
"""Deterministic fake model. Makes no network calls."""
|
|
17
|
+
|
|
18
|
+
name = "mock"
|
|
19
|
+
MODELS = ("echo", "fixed")
|
|
20
|
+
|
|
21
|
+
def __init__(self, replies: dict[str, str] | None = None) -> None:
|
|
22
|
+
"""Create the mock; ``replies`` maps exact prompts to scripted answers."""
|
|
23
|
+
super().__init__(requests_per_minute=None, max_attempts=1)
|
|
24
|
+
self._replies = dict(replies or {})
|
|
25
|
+
|
|
26
|
+
def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
|
|
27
|
+
if model_id not in self.MODELS:
|
|
28
|
+
raise ModelNotFoundError(self.name, model_id)
|
|
29
|
+
default = prompt if model_id == "echo" else "A"
|
|
30
|
+
return Completion(
|
|
31
|
+
text=self._replies.get(prompt, default),
|
|
32
|
+
model_id=f"mock-{model_id}",
|
|
33
|
+
finish_reason="stop",
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
def list_models(self) -> list[str]:
|
|
37
|
+
"""Return the two mock model ids."""
|
|
38
|
+
return list(self.MODELS)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Backend for OpenAI-compatible chat APIs: Groq, OpenRouter and Ollama.
|
|
2
|
+
|
|
3
|
+
All three accept the same ``/chat/completions`` and ``/models`` endpoints,
|
|
4
|
+
so one class covers them; only the base URL and key differ.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from touchstone_llm.backends.base import Completion, HTTPBackend
|
|
12
|
+
from touchstone_llm.config import GenerationSettings
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class OpenAICompatBackend(HTTPBackend):
|
|
16
|
+
"""Talks to any server that implements the OpenAI chat completions API."""
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
*,
|
|
21
|
+
name: str,
|
|
22
|
+
base_url: str,
|
|
23
|
+
api_key: str | None,
|
|
24
|
+
extra_headers: dict[str, str] | None = None,
|
|
25
|
+
**kwargs: Any,
|
|
26
|
+
) -> None:
|
|
27
|
+
"""Create the backend; adds a Bearer token when ``api_key`` is set."""
|
|
28
|
+
headers = dict(extra_headers or {})
|
|
29
|
+
if api_key:
|
|
30
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
31
|
+
super().__init__(name=name, base_url=base_url, api_key=api_key, headers=headers, **kwargs)
|
|
32
|
+
|
|
33
|
+
def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
|
|
34
|
+
data = self._request(
|
|
35
|
+
"POST",
|
|
36
|
+
"/chat/completions",
|
|
37
|
+
model_id=model_id,
|
|
38
|
+
json={
|
|
39
|
+
"model": model_id,
|
|
40
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
41
|
+
"temperature": settings.temperature,
|
|
42
|
+
"top_p": settings.top_p,
|
|
43
|
+
"max_tokens": settings.max_tokens,
|
|
44
|
+
"seed": settings.seed,
|
|
45
|
+
},
|
|
46
|
+
)
|
|
47
|
+
choice = (data.get("choices") or [{}])[0]
|
|
48
|
+
return Completion(
|
|
49
|
+
text=(choice.get("message") or {}).get("content") or "",
|
|
50
|
+
model_id=data.get("model"),
|
|
51
|
+
finish_reason=choice.get("finish_reason"),
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
def list_models(self) -> list[str]:
|
|
55
|
+
"""Return the ids from ``GET /models``, sorted."""
|
|
56
|
+
data = self._request("GET", "/models")
|
|
57
|
+
return sorted(str(m["id"]) for m in data.get("data", []) if "id" in m)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Client-side rate limiting and retries for free-tier APIs.
|
|
2
|
+
|
|
3
|
+
Free tiers allow only a few requests per minute. We space requests out
|
|
4
|
+
ourselves and retry politely when the provider still says "slow down".
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from typing import TypeVar
|
|
12
|
+
|
|
13
|
+
from touchstone_llm.backends.errors import RateLimitError, TransientError
|
|
14
|
+
|
|
15
|
+
T = TypeVar("T")
|
|
16
|
+
|
|
17
|
+
MAX_WAIT_SECONDS = 120.0
|
|
18
|
+
"""Longer waits usually mean a daily quota is used up; we stop instead of hanging."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RateLimiter:
|
|
22
|
+
"""Keep at least ``60 / requests_per_minute`` seconds between requests.
|
|
23
|
+
|
|
24
|
+
``requests_per_minute=None`` disables limiting (used for local Ollama).
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
requests_per_minute: float | None,
|
|
30
|
+
*,
|
|
31
|
+
clock: Callable[[], float] = time.monotonic,
|
|
32
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
33
|
+
) -> None:
|
|
34
|
+
"""Create a limiter; ``clock`` and ``sleep`` are injectable for tests."""
|
|
35
|
+
if requests_per_minute is not None and requests_per_minute <= 0:
|
|
36
|
+
raise ValueError("requests_per_minute must be positive or None")
|
|
37
|
+
self._interval = 60.0 / requests_per_minute if requests_per_minute else 0.0
|
|
38
|
+
self._clock = clock
|
|
39
|
+
self._sleep = sleep
|
|
40
|
+
self._last: float | None = None
|
|
41
|
+
|
|
42
|
+
def acquire(self) -> None:
|
|
43
|
+
"""Block until the next request is allowed."""
|
|
44
|
+
if not self._interval:
|
|
45
|
+
return
|
|
46
|
+
now = self._clock()
|
|
47
|
+
if self._last is not None:
|
|
48
|
+
wait = self._last + self._interval - now
|
|
49
|
+
if wait > 0:
|
|
50
|
+
self._sleep(wait)
|
|
51
|
+
now = self._clock()
|
|
52
|
+
self._last = now
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def with_retries(
|
|
56
|
+
fn: Callable[[], T],
|
|
57
|
+
*,
|
|
58
|
+
max_attempts: int,
|
|
59
|
+
base_delay: float,
|
|
60
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
61
|
+
) -> T:
|
|
62
|
+
"""Call ``fn``; retry on rate limits and transient errors with exponential backoff.
|
|
63
|
+
|
|
64
|
+
Uses the provider's ``Retry-After`` when given. Raises the last error after
|
|
65
|
+
``max_attempts`` tries, or at once if the provider asks for a wait longer
|
|
66
|
+
than ``MAX_WAIT_SECONDS``. Other exceptions are never retried.
|
|
67
|
+
"""
|
|
68
|
+
for attempt in range(1, max_attempts + 1):
|
|
69
|
+
try:
|
|
70
|
+
return fn()
|
|
71
|
+
except (RateLimitError, TransientError) as err:
|
|
72
|
+
if attempt == max_attempts:
|
|
73
|
+
raise
|
|
74
|
+
retry_after = err.retry_after if isinstance(err, RateLimitError) else None
|
|
75
|
+
if retry_after is not None and retry_after > MAX_WAIT_SECONDS:
|
|
76
|
+
raise
|
|
77
|
+
delay = retry_after or min(base_delay * 2 ** (attempt - 1), MAX_WAIT_SECONDS)
|
|
78
|
+
sleep(delay)
|
|
79
|
+
raise AssertionError("unreachable") # pragma: no cover
|
touchstone_llm/cache.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""On-disk cache of model replies (SQLite, standard library only).
|
|
2
|
+
|
|
3
|
+
Free tiers have daily limits and laptops crash. Every reply is stored as
|
|
4
|
+
soon as it arrives, keyed by model, exact prompt and settings, so running
|
|
5
|
+
the same command again resumes where it stopped instead of paying twice.
|
|
6
|
+
The cache stores prompts and replies only, never API keys.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import sqlite3
|
|
14
|
+
from contextlib import closing
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from touchstone_llm.backends.base import Completion
|
|
18
|
+
from touchstone_llm.config import GenerationSettings
|
|
19
|
+
|
|
20
|
+
DEFAULT_CACHE_PATH = Path(".touchstone") / "cache.sqlite"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def cache_key(model: str, prompt: str, settings: GenerationSettings) -> str:
|
|
24
|
+
"""Stable hash of everything that determines a reply."""
|
|
25
|
+
payload = json.dumps(
|
|
26
|
+
{"model": model, "prompt": prompt, "settings": settings.model_dump()},
|
|
27
|
+
sort_keys=True,
|
|
28
|
+
ensure_ascii=False,
|
|
29
|
+
)
|
|
30
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ResponseCache:
|
|
34
|
+
"""Key-value store of :class:`Completion` objects in one SQLite file."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, path: str | Path = DEFAULT_CACHE_PATH) -> None:
|
|
37
|
+
"""Create the file and table if needed."""
|
|
38
|
+
self.path = Path(path)
|
|
39
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
with closing(sqlite3.connect(self.path)) as conn, conn:
|
|
41
|
+
conn.execute(
|
|
42
|
+
"CREATE TABLE IF NOT EXISTS responses ("
|
|
43
|
+
"key TEXT PRIMARY KEY, completion TEXT NOT NULL, "
|
|
44
|
+
"created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def get(self, key: str) -> Completion | None:
|
|
48
|
+
"""Return the cached reply, or ``None``."""
|
|
49
|
+
with closing(sqlite3.connect(self.path)) as conn:
|
|
50
|
+
row = conn.execute("SELECT completion FROM responses WHERE key = ?", (key,)).fetchone()
|
|
51
|
+
return Completion.model_validate_json(row[0]) if row else None
|
|
52
|
+
|
|
53
|
+
def put(self, key: str, completion: Completion) -> None:
|
|
54
|
+
"""Store (or replace) a reply."""
|
|
55
|
+
with closing(sqlite3.connect(self.path)) as conn, conn:
|
|
56
|
+
conn.execute(
|
|
57
|
+
"INSERT OR REPLACE INTO responses (key, completion) VALUES (?, ?)",
|
|
58
|
+
(key, completion.model_dump_json()),
|
|
59
|
+
)
|