touchstone-llm 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. touchstone_llm/__init__.py +3 -0
  2. touchstone_llm/backends/__init__.py +112 -0
  3. touchstone_llm/backends/base.py +207 -0
  4. touchstone_llm/backends/errors.py +41 -0
  5. touchstone_llm/backends/gemini.py +73 -0
  6. touchstone_llm/backends/mock.py +38 -0
  7. touchstone_llm/backends/openai_compat.py +57 -0
  8. touchstone_llm/backends/ratelimit.py +79 -0
  9. touchstone_llm/cache.py +59 -0
  10. touchstone_llm/cli.py +337 -0
  11. touchstone_llm/config.py +83 -0
  12. touchstone_llm/drafts.py +26 -0
  13. touchstone_llm/env.py +146 -0
  14. touchstone_llm/errors.py +11 -0
  15. touchstone_llm/py.typed +0 -0
  16. touchstone_llm/report/__init__.py +84 -0
  17. touchstone_llm/report/templates/report.html.j2 +152 -0
  18. touchstone_llm/report/templates/report.md.j2 +56 -0
  19. touchstone_llm/reproduce.py +63 -0
  20. touchstone_llm/results.py +138 -0
  21. touchstone_llm/review.py +243 -0
  22. touchstone_llm/runner.py +152 -0
  23. touchstone_llm/scoring/__init__.py +1 -0
  24. touchstone_llm/scoring/chrf.py +54 -0
  25. touchstone_llm/scoring/mcq.py +107 -0
  26. touchstone_llm/scoring/normalize.py +37 -0
  27. touchstone_llm/scoring/numeric.py +37 -0
  28. touchstone_llm/scoring/rules.py +176 -0
  29. touchstone_llm/server/__init__.py +1 -0
  30. touchstone_llm/server/app.py +264 -0
  31. touchstone_llm/server/jobs.py +114 -0
  32. touchstone_llm/server/static/assets/fira-code-latin-400-normal-C-QZfXAs.woff +0 -0
  33. touchstone_llm/server/static/assets/fira-code-latin-400-normal-DGosTW8U.woff2 +0 -0
  34. touchstone_llm/server/static/assets/fira-code-latin-500-normal-B6gioGu8.woff2 +0 -0
  35. touchstone_llm/server/static/assets/fira-code-latin-500-normal-aMLC2AMm.woff +0 -0
  36. touchstone_llm/server/static/assets/index-Dgv2BMCR.css +2 -0
  37. touchstone_llm/server/static/assets/index-Pvf3wJWw.js +11 -0
  38. touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-400-normal-BlDd6KY1.woff2 +0 -0
  39. touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-400-normal-CdOJmQ4E.woff +0 -0
  40. touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-500-normal-DO6LFp12.woff +0 -0
  41. touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-500-normal-DVtwyDnM.woff2 +0 -0
  42. touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-600-normal-B5tDaqnD.woff +0 -0
  43. touchstone_llm/server/static/assets/noto-sans-devanagari-devanagari-600-normal-BeyktMLo.woff2 +0 -0
  44. touchstone_llm/server/static/assets/noto-sans-latin-400-normal-BTkUljjl.woff2 +0 -0
  45. touchstone_llm/server/static/assets/noto-sans-latin-400-normal-DPpKfCgK.woff +0 -0
  46. touchstone_llm/server/static/assets/noto-sans-latin-500-normal-CDPngY5C.woff +0 -0
  47. touchstone_llm/server/static/assets/noto-sans-latin-500-normal-DCA2Z9Cj.woff2 +0 -0
  48. touchstone_llm/server/static/assets/noto-sans-latin-600-normal-DVJMj_t0.woff +0 -0
  49. touchstone_llm/server/static/assets/noto-sans-latin-600-normal-Dkgu5AE5.woff2 +0 -0
  50. touchstone_llm/server/static/index.html +22 -0
  51. touchstone_llm/tasks/__init__.py +61 -0
  52. touchstone_llm/tasks/base.py +128 -0
  53. touchstone_llm/tasks/data/drafts/codemixed_instructions.jsonl +25 -0
  54. touchstone_llm/tasks/data/drafts/cultural_context.jsonl +25 -0
  55. touchstone_llm/tasks/data/drafts/hindi_comprehension.jsonl +25 -0
  56. touchstone_llm/tasks/data/drafts/hindi_reasoning.jsonl +25 -0
  57. touchstone_llm/tasks/data/drafts/hinglish_understanding.jsonl +25 -0
  58. touchstone_llm/tasks/data/drafts/translation_hi_en.jsonl +25 -0
  59. touchstone_llm/tasks/instructions.py +38 -0
  60. touchstone_llm/tasks/mcq.py +93 -0
  61. touchstone_llm/tasks/numeric.py +52 -0
  62. touchstone_llm/tasks/translation.py +51 -0
  63. touchstone_llm-0.1.0.dist-info/METADATA +319 -0
  64. touchstone_llm-0.1.0.dist-info/RECORD +67 -0
  65. touchstone_llm-0.1.0.dist-info/WHEEL +4 -0
  66. touchstone_llm-0.1.0.dist-info/entry_points.txt +2 -0
  67. touchstone_llm-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,3 @@
1
+ """touchstone: a touchstone for LLMs in Hindi and Hinglish."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,112 @@
1
+ """Model backends and the provider registry.
2
+
3
+ Built-in providers: ``groq``, ``gemini``, ``openrouter``, ``ollama`` and the
4
+ offline ``mock``. Third-party packages can add more through the
5
+ ``touchstone_llm.backends`` entry point group; each entry point must load a
6
+ callable that takes keyword overrides and returns a :class:`Backend`.
7
+
8
+ Default request rates are cautious guesses for free tiers. Providers change
9
+ their limits, so pass ``requests_per_minute`` to override them.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import os
15
+ from collections.abc import Callable
16
+ from importlib.metadata import entry_points
17
+ from typing import Any
18
+
19
+ from touchstone_llm.backends.base import Backend, Completion
20
+ from touchstone_llm.backends.gemini import GeminiBackend
21
+ from touchstone_llm.backends.mock import MockBackend
22
+ from touchstone_llm.backends.openai_compat import OpenAICompatBackend
23
+ from touchstone_llm.env import get_api_key
24
+ from touchstone_llm.errors import TouchstoneError
25
+
26
+ __all__ = ["Backend", "Completion", "available_backends", "get_backend"]
27
+
28
+ BackendFactory = Callable[..., Backend]
29
+
30
+ DEFAULT_OLLAMA_URL = "http://localhost:11434"
31
+
32
+
33
+ def _groq(**overrides: Any) -> Backend:
34
+ options: dict[str, Any] = {"requests_per_minute": 30, **overrides}
35
+ return OpenAICompatBackend(
36
+ name="groq",
37
+ base_url="https://api.groq.com/openai/v1",
38
+ api_key=get_api_key("groq"),
39
+ **options,
40
+ )
41
+
42
+
43
+ def _openrouter(**overrides: Any) -> Backend:
44
+ options: dict[str, Any] = {"requests_per_minute": 20, **overrides}
45
+ return OpenAICompatBackend(
46
+ name="openrouter",
47
+ base_url="https://openrouter.ai/api/v1",
48
+ api_key=get_api_key("openrouter"),
49
+ extra_headers={"X-Title": "touchstone"},
50
+ **options,
51
+ )
52
+
53
+
54
+ def _ollama(**overrides: Any) -> Backend:
55
+ base = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_URL).strip() or DEFAULT_OLLAMA_URL
56
+ options: dict[str, Any] = {"requests_per_minute": None, "timeout": 300.0, **overrides}
57
+ return OpenAICompatBackend(
58
+ name="ollama",
59
+ base_url=f"{base.rstrip('/')}/v1",
60
+ api_key=None,
61
+ connect_hint="Is Ollama running? Start it with `ollama serve`, then pull a small "
62
+ "model, e.g. `ollama pull <model>`.",
63
+ **options,
64
+ )
65
+
66
+
67
+ def _gemini(**overrides: Any) -> Backend:
68
+ options: dict[str, Any] = {"requests_per_minute": 10, **overrides}
69
+ key = get_api_key("gemini")
70
+ assert key is not None # gemini always needs a key; get_api_key raises otherwise
71
+ return GeminiBackend(api_key=key, **options)
72
+
73
+
74
+ def _mock(**_overrides: Any) -> Backend:
75
+ return MockBackend()
76
+
77
+
78
+ _BUILTINS: dict[str, BackendFactory] = {
79
+ "groq": _groq,
80
+ "gemini": _gemini,
81
+ "openrouter": _openrouter,
82
+ "ollama": _ollama,
83
+ "mock": _mock,
84
+ }
85
+
86
+
87
+ def _plugins() -> dict[str, BackendFactory]:
88
+ found: dict[str, BackendFactory] = {}
89
+ for ep in entry_points(group="touchstone_llm.backends"):
90
+ if ep.name not in _BUILTINS:
91
+ found[ep.name] = ep.load()
92
+ return found
93
+
94
+
95
+ def available_backends() -> list[str]:
96
+ """Names of all built-in and plugin providers, sorted."""
97
+ return sorted({*_BUILTINS, *(ep.name for ep in entry_points(group="touchstone_llm.backends"))})
98
+
99
+
100
+ def get_backend(provider: str, **overrides: Any) -> Backend:
101
+ """Create the backend for ``provider``.
102
+
103
+ Raises:
104
+ TouchstoneError: unknown provider (the message lists known ones).
105
+ MissingAPIKeyError: the provider needs a key that is not set.
106
+
107
+ """
108
+ factory = _BUILTINS.get(provider) or _plugins().get(provider)
109
+ if factory is None:
110
+ known = ", ".join(available_backends())
111
+ raise TouchstoneError(f"Unknown provider '{provider}'. Available providers: {known}.")
112
+ return factory(**overrides)
@@ -0,0 +1,207 @@
1
+ """Backend interface shared by every model provider.
2
+
3
+ To add a provider, subclass :class:`Backend` (or :class:`HTTPBackend`),
4
+ implement ``_generate`` and ``list_models``, and register a factory (see
5
+ ``touchstone_llm.backends``).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import time
11
+ from abc import ABC, abstractmethod
12
+ from collections.abc import Callable
13
+ from typing import Any
14
+
15
+ import httpx
16
+ from pydantic import BaseModel
17
+
18
+ from touchstone_llm import __version__
19
+ from touchstone_llm.backends.errors import (
20
+ AuthError,
21
+ BackendError,
22
+ ModelNotFoundError,
23
+ RateLimitError,
24
+ TransientError,
25
+ )
26
+ from touchstone_llm.backends.ratelimit import RateLimiter, with_retries
27
+ from touchstone_llm.config import GenerationSettings
28
+ from touchstone_llm.env import PROVIDER_KEYS, redact_secrets
29
+
30
+ __all__ = [
31
+ "AuthError",
32
+ "Backend",
33
+ "BackendError",
34
+ "Completion",
35
+ "HTTPBackend",
36
+ "ModelNotFoundError",
37
+ "RateLimitError",
38
+ "TransientError",
39
+ ]
40
+
41
+ _MODEL_MISSING_HINTS = (
42
+ "model_not_found",
43
+ "does not exist",
44
+ "decommissioned",
45
+ "not a valid model",
46
+ "not found",
47
+ "no endpoints found",
48
+ "is not supported",
49
+ )
50
+
51
+
52
+ class Completion(BaseModel):
53
+ """A model's reply to one prompt."""
54
+
55
+ text: str
56
+ model_id: str | None = None
57
+ """Exact model id or version the provider reports, if any."""
58
+ finish_reason: str | None = None
59
+ latency_ms: float = 0.0
60
+
61
+
62
+ class Backend(ABC):
63
+ """Base class: adds rate limiting, retries and timing around ``_generate``."""
64
+
65
+ name: str
66
+
67
+ def __init__(
68
+ self,
69
+ *,
70
+ requests_per_minute: float | None,
71
+ max_attempts: int = 5,
72
+ base_delay: float = 2.0,
73
+ sleep: Callable[[float], None] = time.sleep,
74
+ clock: Callable[[], float] = time.monotonic,
75
+ ) -> None:
76
+ """Set up the rate limiter and retry policy."""
77
+ self._limiter = RateLimiter(requests_per_minute, clock=clock, sleep=sleep)
78
+ self._max_attempts = max_attempts
79
+ self._base_delay = base_delay
80
+ self._sleep = sleep
81
+ self._clock = clock
82
+
83
+ def generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
84
+ """Send one prompt and return the reply, retrying temporary failures."""
85
+
86
+ def attempt() -> Completion:
87
+ self._limiter.acquire()
88
+ start = self._clock()
89
+ completion = self._generate(model_id, prompt, settings)
90
+ elapsed = (self._clock() - start) * 1000
91
+ return completion.model_copy(update={"latency_ms": max(elapsed, 0.0)})
92
+
93
+ return with_retries(
94
+ attempt,
95
+ max_attempts=self._max_attempts,
96
+ base_delay=self._base_delay,
97
+ sleep=self._sleep,
98
+ )
99
+
100
+ @abstractmethod
101
+ def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
102
+ """Make exactly one provider call."""
103
+
104
+ @abstractmethod
105
+ def list_models(self) -> list[str]:
106
+ """Return model ids the provider currently offers, sorted."""
107
+
108
+ def close(self) -> None: # noqa: B027 (optional hook, default does nothing)
109
+ """Release network resources."""
110
+
111
+
112
+ class HTTPBackend(Backend):
113
+ """A backend that talks to a JSON HTTP API with ``httpx``."""
114
+
115
+ def __init__(
116
+ self,
117
+ *,
118
+ name: str,
119
+ base_url: str,
120
+ api_key: str | None,
121
+ headers: dict[str, str] | None = None,
122
+ timeout: float = 60.0,
123
+ connect_hint: str | None = None,
124
+ **kwargs: Any,
125
+ ) -> None:
126
+ """Create the HTTP client. ``connect_hint`` is shown when the server is unreachable."""
127
+ super().__init__(**kwargs)
128
+ self.name = name
129
+ self.base_url = base_url.rstrip("/")
130
+ self._api_key = api_key
131
+ self._connect_hint = connect_hint
132
+ self._client = httpx.Client(
133
+ base_url=self.base_url,
134
+ headers={"User-Agent": f"touchstone/{__version__}", **(headers or {})},
135
+ timeout=timeout,
136
+ )
137
+
138
+ def close(self) -> None:
139
+ """Close the HTTP connection pool."""
140
+ self._client.close()
141
+
142
+ def _redact(self, text: str) -> str:
143
+ text = redact_secrets(text)
144
+ if self._api_key:
145
+ text = text.replace(self._api_key, "***")
146
+ return text
147
+
148
+ def _request(self, method: str, path: str, model_id: str = "", **kwargs: Any) -> Any:
149
+ """Send a request and return parsed JSON, mapping failures to backend errors."""
150
+ try:
151
+ response = self._client.request(method, path, **kwargs)
152
+ except httpx.ConnectError as err:
153
+ hint = f"\n{self._connect_hint}" if self._connect_hint else ""
154
+ raise BackendError(
155
+ f"Could not connect to {self.name} at {self.base_url}.{hint}"
156
+ ) from err
157
+ except httpx.TransportError as err:
158
+ raise TransientError(f"Network problem talking to {self.name}: {err}") from err
159
+ if response.is_success:
160
+ return response.json()
161
+ self._raise_for_status(response, model_id)
162
+
163
+ def _raise_for_status(self, response: httpx.Response, model_id: str) -> None:
164
+ status = response.status_code
165
+ detail = self._redact(_error_detail(response))
166
+ lowered = detail.lower()
167
+ if status in (401, 403) or "api key not valid" in lowered or "api_key_invalid" in lowered:
168
+ key = PROVIDER_KEYS.get(self.name)
169
+ where = f" Check {key.env_var} in your .env file." if key else ""
170
+ raise AuthError(f"{self.name} rejected the API key (HTTP {status}).{where}")
171
+ if status == 429:
172
+ raise RateLimitError(
173
+ f"{self.name} rate limit reached (HTTP 429). {detail}".strip(),
174
+ retry_after=_retry_after(response),
175
+ )
176
+ if status >= 500:
177
+ raise TransientError(f"{self.name} server error (HTTP {status}). {detail}".strip())
178
+ if status == 404 or (status == 400 and any(h in lowered for h in _MODEL_MISSING_HINTS)):
179
+ raise ModelNotFoundError(self.name, model_id, detail)
180
+ raise BackendError(f"{self.name} returned HTTP {status}. {detail}".strip())
181
+
182
+
183
+ def _error_detail(response: httpx.Response) -> str:
184
+ """Best-effort error text from a provider's JSON error body."""
185
+ try:
186
+ data = response.json()
187
+ except ValueError:
188
+ return response.text[:300]
189
+ if isinstance(data, dict):
190
+ error = data.get("error")
191
+ if isinstance(error, dict):
192
+ parts = [str(error.get("message", "")), str(error.get("code", ""))]
193
+ parts += [
194
+ str(d.get("reason", "")) for d in error.get("details", []) if isinstance(d, dict)
195
+ ]
196
+ return " ".join(p for p in parts if p and p != "None")
197
+ if isinstance(error, str):
198
+ return error
199
+ return ""
200
+
201
+
202
+ def _retry_after(response: httpx.Response) -> float | None:
203
+ value = response.headers.get("retry-after")
204
+ try:
205
+ return float(value) if value is not None else None
206
+ except ValueError:
207
+ return None
@@ -0,0 +1,41 @@
1
+ """Errors raised by model backends."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from touchstone_llm.errors import TouchstoneError
6
+
7
+
8
+ class BackendError(TouchstoneError):
9
+ """A model provider call failed in a way that retrying will not fix."""
10
+
11
+
12
+ class AuthError(BackendError):
13
+ """The provider rejected the API key."""
14
+
15
+
16
+ class ModelNotFoundError(BackendError):
17
+ """The model id is unknown, misspelled or retired."""
18
+
19
+ def __init__(self, provider: str, model_id: str, detail: str = "") -> None:
20
+ """Explain that the id may be retired and how to list current ids."""
21
+ msg = (
22
+ f"Model '{model_id}' is not available on {provider} "
23
+ f"(it may be retired or misspelled).\n"
24
+ f"Run `touchstone list-models {provider}` to see current model ids."
25
+ )
26
+ if detail:
27
+ msg += f"\nProvider said: {detail}"
28
+ super().__init__(msg)
29
+
30
+
31
+ class RateLimitError(BackendError):
32
+ """The provider asked us to slow down (HTTP 429). Retryable."""
33
+
34
+ def __init__(self, message: str, retry_after: float | None = None) -> None:
35
+ """Store the provider's suggested wait in seconds, if it sent one."""
36
+ super().__init__(message)
37
+ self.retry_after = retry_after
38
+
39
+
40
+ class TransientError(BackendError):
41
+ """A temporary failure (server error, timeout). Retryable."""
@@ -0,0 +1,73 @@
1
+ """Backend for the Google Gemini API (REST, no SDK).
2
+
3
+ The key is sent in the ``x-goog-api-key`` header, never in the URL, so it
4
+ does not end up in logs or error messages that print URLs.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any
10
+
11
+ from touchstone_llm.backends.base import Completion, HTTPBackend
12
+ from touchstone_llm.config import GenerationSettings
13
+
14
+ GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
15
+
16
+
17
+ class GeminiBackend(HTTPBackend):
18
+ """Calls ``models/{id}:generateContent`` on the Gemini API."""
19
+
20
+ def __init__(self, *, api_key: str, base_url: str = GEMINI_BASE_URL, **kwargs: Any) -> None:
21
+ """Create the backend with the key in a request header."""
22
+ super().__init__(
23
+ name="gemini",
24
+ base_url=base_url,
25
+ api_key=api_key,
26
+ headers={"x-goog-api-key": api_key},
27
+ **kwargs,
28
+ )
29
+
30
+ def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
31
+ model_id = model_id.removeprefix("models/")
32
+ data = self._request(
33
+ "POST",
34
+ f"/models/{model_id}:generateContent",
35
+ model_id=model_id,
36
+ json={
37
+ "contents": [{"role": "user", "parts": [{"text": prompt}]}],
38
+ "generationConfig": {
39
+ "temperature": settings.temperature,
40
+ "topP": settings.top_p,
41
+ "maxOutputTokens": settings.max_tokens,
42
+ "seed": settings.seed,
43
+ },
44
+ },
45
+ )
46
+ candidates = data.get("candidates") or []
47
+ if not candidates:
48
+ block = (data.get("promptFeedback") or {}).get("blockReason")
49
+ return Completion(text="", model_id=data.get("modelVersion"), finish_reason=block)
50
+ first = candidates[0]
51
+ parts = (first.get("content") or {}).get("parts") or []
52
+ text = "".join(p.get("text", "") for p in parts if not p.get("thought"))
53
+ return Completion(
54
+ text=text,
55
+ model_id=data.get("modelVersion"),
56
+ finish_reason=first.get("finishReason"),
57
+ )
58
+
59
+ def list_models(self) -> list[str]:
60
+ """Return ids of models that support ``generateContent``, following pagination."""
61
+ ids: list[str] = []
62
+ token: str | None = None
63
+ while True:
64
+ params: dict[str, str | int] = {"pageSize": 1000}
65
+ if token:
66
+ params["pageToken"] = token
67
+ data = self._request("GET", "/models", params=params)
68
+ for model in data.get("models", []):
69
+ if "generateContent" in model.get("supportedGenerationMethods", []):
70
+ ids.append(str(model["name"]).removeprefix("models/"))
71
+ token = data.get("nextPageToken")
72
+ if not token:
73
+ return sorted(ids)
@@ -0,0 +1,38 @@
1
+ """Offline backend for tests and trying touchstone without any API key.
2
+
3
+ Models:
4
+ ``mock/echo``: replies with the prompt itself.
5
+ ``mock/fixed``: always replies ``"A"``.
6
+ Either can be given scripted ``replies`` keyed by the exact prompt.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from touchstone_llm.backends.base import Backend, Completion, ModelNotFoundError
12
+ from touchstone_llm.config import GenerationSettings
13
+
14
+
15
+ class MockBackend(Backend):
16
+ """Deterministic fake model. Makes no network calls."""
17
+
18
+ name = "mock"
19
+ MODELS = ("echo", "fixed")
20
+
21
+ def __init__(self, replies: dict[str, str] | None = None) -> None:
22
+ """Create the mock; ``replies`` maps exact prompts to scripted answers."""
23
+ super().__init__(requests_per_minute=None, max_attempts=1)
24
+ self._replies = dict(replies or {})
25
+
26
+ def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
27
+ if model_id not in self.MODELS:
28
+ raise ModelNotFoundError(self.name, model_id)
29
+ default = prompt if model_id == "echo" else "A"
30
+ return Completion(
31
+ text=self._replies.get(prompt, default),
32
+ model_id=f"mock-{model_id}",
33
+ finish_reason="stop",
34
+ )
35
+
36
+ def list_models(self) -> list[str]:
37
+ """Return the two mock model ids."""
38
+ return list(self.MODELS)
@@ -0,0 +1,57 @@
1
+ """Backend for OpenAI-compatible chat APIs: Groq, OpenRouter and Ollama.
2
+
3
+ All three accept the same ``/chat/completions`` and ``/models`` endpoints,
4
+ so one class covers them; only the base URL and key differ.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any
10
+
11
+ from touchstone_llm.backends.base import Completion, HTTPBackend
12
+ from touchstone_llm.config import GenerationSettings
13
+
14
+
15
+ class OpenAICompatBackend(HTTPBackend):
16
+ """Talks to any server that implements the OpenAI chat completions API."""
17
+
18
+ def __init__(
19
+ self,
20
+ *,
21
+ name: str,
22
+ base_url: str,
23
+ api_key: str | None,
24
+ extra_headers: dict[str, str] | None = None,
25
+ **kwargs: Any,
26
+ ) -> None:
27
+ """Create the backend; adds a Bearer token when ``api_key`` is set."""
28
+ headers = dict(extra_headers or {})
29
+ if api_key:
30
+ headers["Authorization"] = f"Bearer {api_key}"
31
+ super().__init__(name=name, base_url=base_url, api_key=api_key, headers=headers, **kwargs)
32
+
33
+ def _generate(self, model_id: str, prompt: str, settings: GenerationSettings) -> Completion:
34
+ data = self._request(
35
+ "POST",
36
+ "/chat/completions",
37
+ model_id=model_id,
38
+ json={
39
+ "model": model_id,
40
+ "messages": [{"role": "user", "content": prompt}],
41
+ "temperature": settings.temperature,
42
+ "top_p": settings.top_p,
43
+ "max_tokens": settings.max_tokens,
44
+ "seed": settings.seed,
45
+ },
46
+ )
47
+ choice = (data.get("choices") or [{}])[0]
48
+ return Completion(
49
+ text=(choice.get("message") or {}).get("content") or "",
50
+ model_id=data.get("model"),
51
+ finish_reason=choice.get("finish_reason"),
52
+ )
53
+
54
+ def list_models(self) -> list[str]:
55
+ """Return the ids from ``GET /models``, sorted."""
56
+ data = self._request("GET", "/models")
57
+ return sorted(str(m["id"]) for m in data.get("data", []) if "id" in m)
@@ -0,0 +1,79 @@
1
+ """Client-side rate limiting and retries for free-tier APIs.
2
+
3
+ Free tiers allow only a few requests per minute. We space requests out
4
+ ourselves and retry politely when the provider still says "slow down".
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import time
10
+ from collections.abc import Callable
11
+ from typing import TypeVar
12
+
13
+ from touchstone_llm.backends.errors import RateLimitError, TransientError
14
+
15
+ T = TypeVar("T")
16
+
17
+ MAX_WAIT_SECONDS = 120.0
18
+ """Longer waits usually mean a daily quota is used up; we stop instead of hanging."""
19
+
20
+
21
+ class RateLimiter:
22
+ """Keep at least ``60 / requests_per_minute`` seconds between requests.
23
+
24
+ ``requests_per_minute=None`` disables limiting (used for local Ollama).
25
+ """
26
+
27
+ def __init__(
28
+ self,
29
+ requests_per_minute: float | None,
30
+ *,
31
+ clock: Callable[[], float] = time.monotonic,
32
+ sleep: Callable[[float], None] = time.sleep,
33
+ ) -> None:
34
+ """Create a limiter; ``clock`` and ``sleep`` are injectable for tests."""
35
+ if requests_per_minute is not None and requests_per_minute <= 0:
36
+ raise ValueError("requests_per_minute must be positive or None")
37
+ self._interval = 60.0 / requests_per_minute if requests_per_minute else 0.0
38
+ self._clock = clock
39
+ self._sleep = sleep
40
+ self._last: float | None = None
41
+
42
+ def acquire(self) -> None:
43
+ """Block until the next request is allowed."""
44
+ if not self._interval:
45
+ return
46
+ now = self._clock()
47
+ if self._last is not None:
48
+ wait = self._last + self._interval - now
49
+ if wait > 0:
50
+ self._sleep(wait)
51
+ now = self._clock()
52
+ self._last = now
53
+
54
+
55
+ def with_retries(
56
+ fn: Callable[[], T],
57
+ *,
58
+ max_attempts: int,
59
+ base_delay: float,
60
+ sleep: Callable[[float], None] = time.sleep,
61
+ ) -> T:
62
+ """Call ``fn``; retry on rate limits and transient errors with exponential backoff.
63
+
64
+ Uses the provider's ``Retry-After`` when given. Raises the last error after
65
+ ``max_attempts`` tries, or at once if the provider asks for a wait longer
66
+ than ``MAX_WAIT_SECONDS``. Other exceptions are never retried.
67
+ """
68
+ for attempt in range(1, max_attempts + 1):
69
+ try:
70
+ return fn()
71
+ except (RateLimitError, TransientError) as err:
72
+ if attempt == max_attempts:
73
+ raise
74
+ retry_after = err.retry_after if isinstance(err, RateLimitError) else None
75
+ if retry_after is not None and retry_after > MAX_WAIT_SECONDS:
76
+ raise
77
+ delay = retry_after or min(base_delay * 2 ** (attempt - 1), MAX_WAIT_SECONDS)
78
+ sleep(delay)
79
+ raise AssertionError("unreachable") # pragma: no cover
@@ -0,0 +1,59 @@
1
+ """On-disk cache of model replies (SQLite, standard library only).
2
+
3
+ Free tiers have daily limits and laptops crash. Every reply is stored as
4
+ soon as it arrives, keyed by model, exact prompt and settings, so running
5
+ the same command again resumes where it stopped instead of paying twice.
6
+ The cache stores prompts and replies only, never API keys.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ import json
13
+ import sqlite3
14
+ from contextlib import closing
15
+ from pathlib import Path
16
+
17
+ from touchstone_llm.backends.base import Completion
18
+ from touchstone_llm.config import GenerationSettings
19
+
20
+ DEFAULT_CACHE_PATH = Path(".touchstone") / "cache.sqlite"
21
+
22
+
23
+ def cache_key(model: str, prompt: str, settings: GenerationSettings) -> str:
24
+ """Stable hash of everything that determines a reply."""
25
+ payload = json.dumps(
26
+ {"model": model, "prompt": prompt, "settings": settings.model_dump()},
27
+ sort_keys=True,
28
+ ensure_ascii=False,
29
+ )
30
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
31
+
32
+
33
+ class ResponseCache:
34
+ """Key-value store of :class:`Completion` objects in one SQLite file."""
35
+
36
+ def __init__(self, path: str | Path = DEFAULT_CACHE_PATH) -> None:
37
+ """Create the file and table if needed."""
38
+ self.path = Path(path)
39
+ self.path.parent.mkdir(parents=True, exist_ok=True)
40
+ with closing(sqlite3.connect(self.path)) as conn, conn:
41
+ conn.execute(
42
+ "CREATE TABLE IF NOT EXISTS responses ("
43
+ "key TEXT PRIMARY KEY, completion TEXT NOT NULL, "
44
+ "created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)"
45
+ )
46
+
47
+ def get(self, key: str) -> Completion | None:
48
+ """Return the cached reply, or ``None``."""
49
+ with closing(sqlite3.connect(self.path)) as conn:
50
+ row = conn.execute("SELECT completion FROM responses WHERE key = ?", (key,)).fetchone()
51
+ return Completion.model_validate_json(row[0]) if row else None
52
+
53
+ def put(self, key: str, completion: Completion) -> None:
54
+ """Store (or replace) a reply."""
55
+ with closing(sqlite3.connect(self.path)) as conn, conn:
56
+ conn.execute(
57
+ "INSERT OR REPLACE INTO responses (key, completion) VALUES (?, ?)",
58
+ (key, completion.model_dump_json()),
59
+ )