cairnkit 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cairnkit/llm/errors.py ADDED
@@ -0,0 +1,185 @@
1
+ # Adapted from hermes-agent v0.23.0 (2bd378e): agent/error_classifier.py and
2
+ # agent/timeout_error_summary.py
3
+ # Copyright (c) 2025 Nous Research. MIT License. Condensed to OpenAI-compatible endpoints.
4
+ """Classify model API failures into a recovery decision.
5
+
6
+ The retry/fallback code consults the hints on :class:`ClassifiedError` instead of
7
+ re-matching error strings itself (same contract as Hermes' classifier).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import enum
13
+ import re
14
+ from dataclasses import dataclass, field
15
+ from typing import Any
16
+
17
+ import httpx
18
+
19
+
20
+ class FailoverReason(enum.Enum):
21
+ auth = "auth" # 401/403 credential refused: abort, retrying will not help
22
+ billing = "billing" # 402 or credit exhausted: fall back, do not retry this model
23
+ rate_limit = "rate_limit" # 429 from the gateway itself: back off, then fall back
24
+ upstream_rate_limit = "upstream_rate_limit" # aggregator's upstream 429: key is fine, fall back
25
+ overloaded = "overloaded" # 503/529: back off and retry
26
+ server_error = "server_error" # 500/502: retry
27
+ timeout = "timeout" # connect/read timeout: retry
28
+ context_overflow = "context_overflow" # prompt too long: compress, do not fail over
29
+ payload_too_large = "payload_too_large" # 413, or a 400 naming a body byte cap: compress
30
+ model_not_found = "model_not_found" # 404 / invalid model id: fall back to another model
31
+ provider_policy_blocked = "provider_policy_blocked" # data policy excluded every endpoint
32
+ content_policy_blocked = "content_policy_blocked" # safety filter: same input fails again
33
+ format_error = "format_error" # 400 malformed request: abort
34
+ unknown = "unknown"
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class ClassifiedError:
39
+ reason: FailoverReason
40
+ status_code: int | None = None
41
+ message: str = ""
42
+ retryable: bool = False
43
+ should_fallback: bool = False
44
+ should_compress: bool = False
45
+ context: dict[str, Any] = field(default_factory=dict)
46
+
47
+
48
+ class ModelAPIError(Exception):
49
+ """Non-2xx answer from a model endpoint. ``body`` is the parsed JSON when available."""
50
+
51
+ def __init__(self, status_code: int, message: str, body: Any = None):
52
+ super().__init__(f"HTTP {status_code}: {message}")
53
+ self.status_code = status_code
54
+ self.message = message
55
+ self.body = body
56
+
57
+
58
+ _CONTEXT_PATTERNS = (
59
+ "context length",
60
+ "context_length",
61
+ "maximum context",
62
+ "too many tokens",
63
+ "prompt is too long",
64
+ "reduce the length",
65
+ "context window",
66
+ )
67
+ _BILLING_PATTERNS = ("insufficient credits", "insufficient_quota", "key limit exceeded", "credit")
68
+ _POLICY_PATTERNS = ("data policy", "no endpoints found matching your data policy")
69
+ _CONTENT_PATTERNS = ("content policy", "flagged", "moderation", "safety")
70
+
71
+
72
+ # Hermes v0.23.0: byte-limited gateways report an oversized request BODY as a 400 (Novita
73
+ # "request body too large, must be less than 20MB", Z.AI "Request size exceeds the limit of
74
+ # ..."). Compressing helps; retrying the same body does not. A subject AND an overflow word are
75
+ # both required, so field-length complaints stay out; a cap counted in tokens keeps its rules.
76
+ _BYTE_SIZE_SUBJECT = re.compile(
77
+ r"(?<!response )\b(?:(?:request ?size|(?:request )?(?:body|payload) size)(?: is)? "
78
+ r"(?:exceed|too large|too big)"
79
+ r"|request body(?: is)? too large|(?:request )?payload(?: is)? too large)"
80
+ )
81
+ _BYTE_SIZE_OVERFLOW = re.compile(r"exceed|too large|too big")
82
+ _TOKEN_COUNT = re.compile(r"\btokens?\b")
83
+ _REQUEST_WORDS = {"request", "requestsize", "byte", "bytes"}
84
+
85
+ # Hermes v0.23.0 (timeout_error_summary): say whether a connection was ever made and what to
86
+ # change, instead of the transport's bare "timed out". Both keep the "timed out" wording.
87
+ _CONNECT_TIMEOUT = (
88
+ "Connection to the provider timed out before it was established "
89
+ "(check the base_url and network path)"
90
+ )
91
+ _READ_TIMEOUT = (
92
+ "Request timed out waiting for the provider to respond (if the model needs longer, "
93
+ "raise timeout= on the model client)"
94
+ )
95
+
96
+
97
+ def _is_byte_size_limit_message(message: str) -> bool:
98
+ msg = message.lower()
99
+ if not _BYTE_SIZE_OVERFLOW.search(msg) or _TOKEN_COUNT.search(msg):
100
+ return False
101
+ if _BYTE_SIZE_SUBJECT.search(msg):
102
+ return True
103
+ # "content length" alone is a field complaint unless the request or bytes are named. "_"
104
+ # stays inside a word, so the body's "invalid_request_error" type is not "request".
105
+ content_length = "content length" in msg or "content-length" in msg
106
+ return content_length and bool(_REQUEST_WORDS & set(re.split(r"[^a-z0-9_]+", msg)))
107
+
108
+
109
+ def _timeout(error: httpx.TimeoutException) -> ClassifiedError:
110
+ # Pool and write timeouts: the provider never got to answer, so neither hint applies.
111
+ if isinstance(error, httpx.ConnectTimeout):
112
+ phase, message = "connect", _CONNECT_TIMEOUT
113
+ elif isinstance(error, httpx.ReadTimeout):
114
+ phase, message = "read", _READ_TIMEOUT
115
+ else:
116
+ phase, message = None, str(error)
117
+ return ClassifiedError(
118
+ FailoverReason.timeout,
119
+ message=message,
120
+ retryable=True,
121
+ should_fallback=True,
122
+ context={"timeout_phase": phase} if phase else {},
123
+ )
124
+
125
+
126
+ def _text(error: ModelAPIError) -> str:
127
+ return f"{error.message} {error.body!r}".lower()
128
+
129
+
130
+ def _upstream_429(body: Any) -> bool:
131
+ """OpenRouter wraps provider errors as ``{"error": {"metadata": {"provider_name": ...}}}``."""
132
+ if not isinstance(body, dict):
133
+ return False
134
+ meta = (
135
+ (body.get("error") or {}).get("metadata") if isinstance(body.get("error"), dict) else None
136
+ )
137
+ return isinstance(meta, dict) and bool(meta.get("provider_name") or meta.get("raw"))
138
+
139
+
140
+ def classify_error(error: BaseException) -> ClassifiedError:
141
+ if isinstance(error, httpx.TimeoutException):
142
+ return _timeout(error)
143
+ if isinstance(error, httpx.TransportError):
144
+ return ClassifiedError(
145
+ FailoverReason.server_error, message=str(error), retryable=True, should_fallback=True
146
+ )
147
+ if not isinstance(error, ModelAPIError):
148
+ return ClassifiedError(FailoverReason.unknown, message=type(error).__name__)
149
+
150
+ status, text = error.status_code, _text(error)
151
+
152
+ def verdict(reason: FailoverReason, **hints: Any) -> ClassifiedError:
153
+ return ClassifiedError(reason, status_code=status, message=error.message, **hints)
154
+
155
+ if any(p in text for p in _CONTEXT_PATTERNS):
156
+ return verdict(FailoverReason.context_overflow, should_compress=True)
157
+ if status == 413:
158
+ return verdict(FailoverReason.payload_too_large, should_compress=True)
159
+ if status == 402 or any(p in text for p in _BILLING_PATTERNS):
160
+ return verdict(FailoverReason.billing, should_fallback=True)
161
+ if status == 401:
162
+ return verdict(FailoverReason.auth)
163
+ if status == 403:
164
+ if any(p in text for p in _POLICY_PATTERNS):
165
+ return verdict(FailoverReason.provider_policy_blocked, should_fallback=True)
166
+ return verdict(FailoverReason.auth)
167
+ if status == 404:
168
+ if any(p in text for p in _POLICY_PATTERNS):
169
+ return verdict(FailoverReason.provider_policy_blocked, should_fallback=True)
170
+ return verdict(FailoverReason.model_not_found, should_fallback=True)
171
+ if status == 429:
172
+ if _upstream_429(error.body):
173
+ return verdict(FailoverReason.upstream_rate_limit, should_fallback=True)
174
+ return verdict(FailoverReason.rate_limit, retryable=True, should_fallback=True)
175
+ if status in (503, 529):
176
+ return verdict(FailoverReason.overloaded, retryable=True, should_fallback=True)
177
+ if 500 <= status < 600:
178
+ return verdict(FailoverReason.server_error, retryable=True, should_fallback=True)
179
+ if status == 400:
180
+ if _is_byte_size_limit_message(error.message):
181
+ return verdict(FailoverReason.payload_too_large, should_compress=True)
182
+ if any(p in text for p in _CONTENT_PATTERNS):
183
+ return verdict(FailoverReason.content_policy_blocked)
184
+ return verdict(FailoverReason.format_error)
185
+ return verdict(FailoverReason.unknown)
@@ -0,0 +1,107 @@
1
+ # Adapted from hermes-agent v0.23.0 (2bd378e): agent/fallback_cooldown.py, the fallback walk
2
+ # in agent/chat_completion_helpers.py and agent/timeout_error_summary.py.
3
+ # Copyright (c) 2025 Nous Research. MIT License.
4
+ """Try a chain of models; retry transient errors, fall back on model-level failures."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import asyncio
9
+ import logging
10
+ import time
11
+ from collections.abc import Awaitable, Callable
12
+ from typing import Any
13
+
14
+ from cairnkit.llm.base import ChatModel, emit_delta
15
+ from cairnkit.llm.errors import ClassifiedError, FailoverReason, classify_error
16
+ from cairnkit.types import Message, ModelResponse
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ Sleep = Callable[[float], Awaitable[None]]
21
+
22
+
23
+ def _label(err: ClassifiedError) -> str:
24
+ """The failure class, plus which phase timed out (Hermes v0.23.0 timeout_error_summary)."""
25
+ if err.context.get("timeout_phase"):
26
+ return f"{err.reason.value} ({err.message})"
27
+ return err.reason.value
28
+
29
+
30
+ class AllModelsFailed(Exception):
31
+ def __init__(self, errors: list[tuple[str, ClassifiedError]]):
32
+ summary = "; ".join(f"{name}: {_label(err)}" for name, err in errors)
33
+ super().__init__(f"every model failed ({summary})")
34
+ self.errors = errors
35
+
36
+
37
+ class FallbackModel:
38
+ """Primary model first; on fallback-worthy failures move down the chain.
39
+
40
+ A model that hit a rate limit is skipped for ``cooldown_seconds`` so later calls
41
+ go straight to the next model instead of waiting on a throttled one.
42
+ Errors that no other model can fix (auth, malformed request, context overflow)
43
+ are raised immediately.
44
+ """
45
+
46
+ def __init__(
47
+ self,
48
+ models: list[ChatModel],
49
+ *,
50
+ retries: int = 2,
51
+ backoff_seconds: float = 1.0,
52
+ cooldown_seconds: float = 60.0,
53
+ sleep: Sleep = asyncio.sleep,
54
+ clock: Callable[[], float] = time.monotonic,
55
+ ):
56
+ if not models:
57
+ raise ValueError("at least one model is required")
58
+ self.models = models
59
+ self.name = models[0].name
60
+ self._retries = retries
61
+ self._backoff = backoff_seconds
62
+ self._cooldown = cooldown_seconds
63
+ self._sleep = sleep
64
+ self._clock = clock
65
+ self._cooling_until: dict[str, float] = {}
66
+
67
+ async def complete(self, messages: list[Message], **kwargs: Any) -> ModelResponse:
68
+ """Streamed text cannot be taken back, so once a delta went out, errors are raised."""
69
+ errors: list[tuple[str, ClassifiedError]] = []
70
+ user_delta = kwargs.pop("on_delta", None)
71
+ streamed = False
72
+
73
+ async def on_delta(text: str) -> None:
74
+ nonlocal streamed
75
+ streamed = True
76
+ await emit_delta(user_delta, text)
77
+
78
+ if user_delta is not None:
79
+ kwargs["on_delta"] = on_delta
80
+ now = self._clock()
81
+ candidates = [m for m in self.models if self._cooling_until.get(m.name, 0) <= now]
82
+ for model in candidates or self.models:
83
+ for attempt in range(self._retries + 1):
84
+ try:
85
+ return await model.complete(messages, **kwargs)
86
+ except Exception as exc: # noqa: BLE001 - classified below
87
+ if streamed:
88
+ raise
89
+ verdict = classify_error(exc)
90
+ logger.warning("model %s failed: %s", model.name, verdict.reason.value)
91
+ if verdict.reason in (
92
+ FailoverReason.rate_limit,
93
+ FailoverReason.upstream_rate_limit,
94
+ ):
95
+ self._cooling_until[model.name] = self._clock() + self._cooldown
96
+ if not verdict.retryable and not verdict.should_fallback:
97
+ raise
98
+ if (
99
+ verdict.retryable
100
+ and attempt < self._retries
101
+ and verdict.reason not in (FailoverReason.rate_limit,)
102
+ ):
103
+ await self._sleep(self._backoff * (2**attempt))
104
+ continue
105
+ errors.append((model.name, verdict))
106
+ break
107
+ raise AllModelsFailed(errors)
@@ -0,0 +1,281 @@
1
+ # Adapted from hermes-agent v0.23.0 (2bd378e): agent/chat_completion_helpers_tool_calls.py
2
+ # (streamed tool-call assembly). Copyright (c) 2025 Nous Research. MIT License.
3
+ """Client for any OpenAI-compatible ``/chat/completions`` endpoint."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ from typing import Any
9
+
10
+ import httpx
11
+
12
+ from cairnkit.llm.base import DeltaHandler, emit_delta
13
+ from cairnkit.llm.errors import ModelAPIError
14
+ from cairnkit.types import Message, ModelResponse, ToolCall, Usage
15
+
16
+ OPENROUTER_URL = "https://openrouter.ai/api/v1"
17
+
18
+
19
+ class OpenAICompatModel:
20
+ def __init__(
21
+ self,
22
+ model: str,
23
+ *,
24
+ base_url: str,
25
+ api_key: str,
26
+ extra_body: dict[str, Any] | None = None,
27
+ headers: dict[str, str] | None = None,
28
+ timeout: float = 60.0,
29
+ transport: httpx.AsyncBaseTransport | None = None,
30
+ ):
31
+ if not api_key:
32
+ raise ValueError("api_key is required")
33
+ self.name = model
34
+ self._url = base_url.rstrip("/") + "/chat/completions"
35
+ self._headers = {"Authorization": f"Bearer {api_key}", **(headers or {})}
36
+ self._extra_body = extra_body or {}
37
+ self._timeout = timeout
38
+ self._transport = transport
39
+
40
+ async def complete(
41
+ self,
42
+ messages: list[Message],
43
+ *,
44
+ tools: list[dict[str, Any]] | None = None,
45
+ temperature: float | None = None,
46
+ max_tokens: int | None = None,
47
+ on_delta: DeltaHandler | None = None,
48
+ ) -> ModelResponse:
49
+ body: dict[str, Any] = {"model": self.name, "messages": messages, **self._extra_body}
50
+ if tools:
51
+ body["tools"] = tools
52
+ if temperature is not None:
53
+ body["temperature"] = temperature
54
+ if max_tokens is not None:
55
+ body["max_tokens"] = max_tokens
56
+ if on_delta is not None:
57
+ return await self._stream(body, on_delta)
58
+ async with httpx.AsyncClient(timeout=self._timeout, transport=self._transport) as client:
59
+ response = await client.post(self._url, json=body, headers=self._headers)
60
+ payload = _json(response)
61
+ if response.status_code >= 400:
62
+ raise ModelAPIError(response.status_code, _error_message(payload, response), payload)
63
+ # Some gateways answer 200 with an error object instead of choices.
64
+ if isinstance(payload, dict) and payload.get("error") and not payload.get("choices"):
65
+ error = payload["error"]
66
+ code = error.get("code") if isinstance(error, dict) else None
67
+ raise ModelAPIError(
68
+ code if isinstance(code, int) else 502, _error_message(payload, response), payload
69
+ )
70
+ return _parse(payload, self.name)
71
+
72
+ async def _stream(self, body: dict[str, Any], on_delta: DeltaHandler) -> ModelResponse:
73
+ body = {**body, "stream": True, "stream_options": {"include_usage": True}}
74
+ content: list[str] = []
75
+ calls = _ToolCallAccumulator()
76
+ final: dict[str, Any] = {"model": self.name, "finish_reason": None, "usage": {}}
77
+ async with (
78
+ httpx.AsyncClient(timeout=self._timeout, transport=self._transport) as client,
79
+ client.stream("POST", self._url, json=body, headers=self._headers) as response,
80
+ ):
81
+ if response.status_code >= 400:
82
+ await response.aread()
83
+ payload = _json(response)
84
+ raise ModelAPIError(
85
+ response.status_code, _error_message(payload, response), payload
86
+ )
87
+ async for line in response.aiter_lines():
88
+ if not line.startswith("data:"):
89
+ continue # SSE comments (": OPENROUTER PROCESSING") and blank lines
90
+ data = line[5:].strip()
91
+ if data == "[DONE]":
92
+ break
93
+ try:
94
+ chunk = json.loads(data)
95
+ except ValueError:
96
+ continue
97
+ if isinstance(chunk.get("error"), dict):
98
+ error = chunk["error"]
99
+ code = error.get("code") if isinstance(error.get("code"), int) else 502
100
+ raise ModelAPIError(code, str(error.get("message", "stream error")), chunk)
101
+ final["model"] = chunk.get("model") or final["model"]
102
+ if chunk.get("usage"):
103
+ final["usage"] = chunk["usage"]
104
+ for choice in chunk.get("choices") or []:
105
+ delta = choice.get("delta") or {}
106
+ if text := delta.get("content"):
107
+ content.append(text)
108
+ await emit_delta(on_delta, text)
109
+ for raw in delta.get("tool_calls") or []:
110
+ calls.feed(raw)
111
+ final["finish_reason"] = choice.get("finish_reason") or final["finish_reason"]
112
+ message = {
113
+ "content": "".join(content) or None,
114
+ "tool_calls": [
115
+ {
116
+ "id": c["id"] or f"call_{i}",
117
+ "function": {"name": c["name"], "arguments": "".join(c["args"])},
118
+ }
119
+ for i, c in sorted(calls.slots.items())
120
+ ],
121
+ }
122
+ return _parse(
123
+ {
124
+ "model": final["model"],
125
+ "usage": final["usage"],
126
+ "choices": [{"message": message, "finish_reason": final["finish_reason"]}],
127
+ },
128
+ self.name,
129
+ )
130
+
131
+
132
+ class _ToolCallAccumulator:
133
+ """Assemble streamed tool-call deltas into whole calls (slot -> id, name, argument parts).
134
+
135
+ Ollama-compatible endpoints reuse index 0 for every call of a parallel batch, told apart
136
+ only by id, so a new id at a seen index can open a new slot. LM Studio and Kimi send a fresh
137
+ id on every argument fragment without a name, and some MiniMax routes resend the same name
138
+ with each fragment: only a named delta opens a call, and one with the current call's name
139
+ only after that call's arguments have closed.
140
+ """
141
+
142
+ def __init__(self) -> None:
143
+ self.slots: dict[int, dict[str, Any]] = {}
144
+ self._slot_by_index: dict[int, int] = {}
145
+ self._last_id: dict[int, str] = {}
146
+ # slot -> (fragments scanned, depth, in_string, escaped, started)
147
+ self._scan: dict[int, tuple[int, int, bool, bool, bool]] = {}
148
+
149
+ def feed(self, raw: dict[str, Any]) -> None:
150
+ index = raw.get("index")
151
+ index = index if isinstance(index, int) else 0
152
+ call_id = str(raw.get("id") or "")
153
+ function = raw.get("function") or {}
154
+ slot = self._slot_by_index.setdefault(index, index)
155
+ last = self._last_id.get(index)
156
+ if (
157
+ call_id
158
+ and last is not None
159
+ and call_id != last
160
+ and self._starts_new_call(function, slot)
161
+ ):
162
+ slot = self._slot_by_index[index] = max(self.slots, default=-1) + 1
163
+ if call_id:
164
+ self._last_id[index] = call_id
165
+ entry = self.slots.setdefault(slot, {"id": "", "name": "", "args": []})
166
+ entry["id"] = call_id or entry["id"]
167
+ # Assignment, not +=: some providers resend the full name with every chunk.
168
+ entry["name"] = function.get("name") or entry["name"]
169
+ if function.get("arguments"):
170
+ entry["args"].append(function["arguments"])
171
+
172
+ def _starts_new_call(self, function: dict[str, Any], slot: int) -> bool:
173
+ name = function.get("name")
174
+ if not name:
175
+ return False
176
+ if name != self.slots.get(slot, {}).get("name"):
177
+ return True
178
+ return self._arguments_closed(slot)
179
+
180
+ def _arguments_closed(self, slot: int) -> bool:
181
+ """Brackets balanced outside strings (not strict JSON, so a repairable call still ends
182
+ its slot). Scans only fragments added since the last check."""
183
+ parts = self.slots.get(slot, {}).get("args", [])
184
+ done, depth, in_string, escaped, started = self._scan.get(slot, (0, 0, False, False, False))
185
+ for ch in "".join(parts[done:]):
186
+ if in_string:
187
+ if escaped:
188
+ escaped = False
189
+ elif ch == "\\":
190
+ escaped = True
191
+ elif ch == '"':
192
+ in_string = False
193
+ elif ch == '"':
194
+ in_string = True
195
+ elif ch in "{[":
196
+ depth, started = depth + 1, True
197
+ elif ch in "}]":
198
+ depth -= 1
199
+ self._scan[slot] = (len(parts), depth, in_string, escaped, started)
200
+ return started and depth == 0 and not in_string
201
+
202
+
203
+ def openrouter(
204
+ model: str,
205
+ *,
206
+ api_key: str,
207
+ deny_data_collection: bool = True,
208
+ sort: str | None = "throughput",
209
+ only_providers: list[str] | None = None,
210
+ app_name: str | None = None,
211
+ timeout: float = 60.0,
212
+ transport: httpx.AsyncBaseTransport | None = None,
213
+ ) -> OpenAICompatModel:
214
+ """OpenRouter with privacy-first routing: providers that store/train on data are excluded."""
215
+ provider: dict[str, Any] = {}
216
+ if deny_data_collection:
217
+ provider["data_collection"] = "deny"
218
+ if sort:
219
+ provider["sort"] = sort
220
+ if only_providers:
221
+ provider["only"] = only_providers
222
+ headers = {"X-Title": app_name} if app_name else None
223
+ return OpenAICompatModel(
224
+ model,
225
+ base_url=OPENROUTER_URL,
226
+ api_key=api_key,
227
+ extra_body={"provider": provider, "usage": {"include": True}} if provider else None,
228
+ headers=headers,
229
+ timeout=timeout,
230
+ transport=transport,
231
+ )
232
+
233
+
234
+ def _json(response: httpx.Response) -> Any:
235
+ try:
236
+ return response.json()
237
+ except ValueError:
238
+ return None
239
+
240
+
241
+ def _error_message(payload: Any, response: httpx.Response) -> str:
242
+ if isinstance(payload, dict) and isinstance(payload.get("error"), dict):
243
+ return str(payload["error"].get("message") or response.reason_phrase)
244
+ return response.reason_phrase or "model request failed"
245
+
246
+
247
+ def _parse(payload: Any, fallback_model: str) -> ModelResponse:
248
+ try:
249
+ choice = payload["choices"][0]
250
+ message = choice["message"]
251
+ except (KeyError, IndexError, TypeError) as exc:
252
+ raise ModelAPIError(502, "malformed completion payload", payload) from exc
253
+ calls = []
254
+ for raw in message.get("tool_calls") or []:
255
+ function = raw.get("function") or {}
256
+ arguments = function.get("arguments")
257
+ if not isinstance(arguments, str):
258
+ arguments = json.dumps(arguments or {})
259
+ calls.append(
260
+ ToolCall(
261
+ id=raw.get("id") or f"call_{len(calls)}",
262
+ name=function.get("name", ""),
263
+ arguments=arguments,
264
+ )
265
+ )
266
+ usage_raw = payload.get("usage") or {}
267
+ details = usage_raw.get("prompt_tokens_details") or {}
268
+ usage = Usage(
269
+ prompt_tokens=int(usage_raw.get("prompt_tokens") or 0),
270
+ completion_tokens=int(usage_raw.get("completion_tokens") or 0),
271
+ cached_tokens=int(details.get("cached_tokens") or 0),
272
+ cost=float(usage_raw["cost"]) if isinstance(usage_raw.get("cost"), int | float) else None,
273
+ calls=1,
274
+ )
275
+ return ModelResponse(
276
+ content=message.get("content"),
277
+ tool_calls=calls,
278
+ usage=usage,
279
+ model=payload.get("model") or fallback_model,
280
+ finish_reason=choice.get("finish_reason"),
281
+ )
@@ -0,0 +1,54 @@
1
+ """Deterministic model for tests, demos and offline development."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable
6
+ from typing import Any
7
+
8
+ from cairnkit.llm.base import DeltaHandler, emit_delta
9
+ from cairnkit.types import Message, ModelResponse, Usage
10
+
11
+ Responder = Callable[[list[Message], list[dict[str, Any]] | None], ModelResponse]
12
+
13
+
14
+ class ScriptedModel:
15
+ """Answers from a list of canned responses, or by calling ``responder``."""
16
+
17
+ def __init__(
18
+ self,
19
+ responses: list[ModelResponse] | None = None,
20
+ *,
21
+ responder: Responder | None = None,
22
+ name: str = "scripted",
23
+ ):
24
+ if (responses is None) == (responder is None):
25
+ raise ValueError("pass exactly one of responses or responder")
26
+ self.name = name
27
+ self._responses = list(responses or [])
28
+ self._responder = responder
29
+ self.requests: list[dict[str, Any]] = []
30
+
31
+ async def complete(
32
+ self,
33
+ messages: list[Message],
34
+ *,
35
+ tools: list[dict[str, Any]] | None = None,
36
+ temperature: float | None = None,
37
+ max_tokens: int | None = None,
38
+ on_delta: DeltaHandler | None = None,
39
+ ) -> ModelResponse:
40
+ self.requests.append({"messages": [dict(m) for m in messages], "tools": tools})
41
+ if self._responder is not None:
42
+ response = self._responder(messages, tools)
43
+ else:
44
+ if not self._responses:
45
+ raise RuntimeError("ScriptedModel ran out of responses")
46
+ response = self._responses.pop(0)
47
+ if response.usage.calls == 0:
48
+ response.usage = Usage(calls=1)
49
+ response.model = response.model or self.name
50
+ if on_delta is not None and response.content:
51
+ words = response.content.split(" ")
52
+ for i, word in enumerate(words):
53
+ await emit_delta(on_delta, word if i == len(words) - 1 else word + " ")
54
+ return response