cairnkit 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cairnkit/__init__.py +143 -0
- cairnkit/agents.py +111 -0
- cairnkit/compression.py +973 -0
- cairnkit/curator.py +217 -0
- cairnkit/delegate.py +99 -0
- cairnkit/documents.py +421 -0
- cairnkit/guardrails.py +124 -0
- cairnkit/llm/__init__.py +17 -0
- cairnkit/llm/base.py +34 -0
- cairnkit/llm/errors.py +185 -0
- cairnkit/llm/fallback.py +107 -0
- cairnkit/llm/openai_compat.py +281 -0
- cairnkit/llm/scripted.py +54 -0
- cairnkit/loop.py +310 -0
- cairnkit/memory.py +185 -0
- cairnkit/memory_ops.py +201 -0
- cairnkit/pricing.py +37 -0
- cairnkit/prompt.py +27 -0
- cairnkit/py.typed +0 -0
- cairnkit/review.py +55 -0
- cairnkit/search.py +130 -0
- cairnkit/security.py +215 -0
- cairnkit/skills.py +184 -0
- cairnkit/titles.py +75 -0
- cairnkit/tools.py +175 -0
- cairnkit/types.py +58 -0
- cairnkit-0.5.0.dist-info/METADATA +234 -0
- cairnkit-0.5.0.dist-info/RECORD +31 -0
- cairnkit-0.5.0.dist-info/WHEEL +4 -0
- cairnkit-0.5.0.dist-info/licenses/LICENSE +22 -0
- cairnkit-0.5.0.dist-info/licenses/NOTICE.md +31 -0
cairnkit/llm/errors.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# Adapted from hermes-agent v0.23.0 (2bd378e): agent/error_classifier.py and
|
|
2
|
+
# agent/timeout_error_summary.py
|
|
3
|
+
# Copyright (c) 2025 Nous Research. MIT License. Condensed to OpenAI-compatible endpoints.
|
|
4
|
+
"""Classify model API failures into a recovery decision.
|
|
5
|
+
|
|
6
|
+
The retry/fallback code consults the hints on :class:`ClassifiedError` instead of
|
|
7
|
+
re-matching error strings itself (same contract as Hermes' classifier).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import enum
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FailoverReason(enum.Enum):
|
|
21
|
+
auth = "auth" # 401/403 credential refused: abort, retrying will not help
|
|
22
|
+
billing = "billing" # 402 or credit exhausted: fall back, do not retry this model
|
|
23
|
+
rate_limit = "rate_limit" # 429 from the gateway itself: back off, then fall back
|
|
24
|
+
upstream_rate_limit = "upstream_rate_limit" # aggregator's upstream 429: key is fine, fall back
|
|
25
|
+
overloaded = "overloaded" # 503/529: back off and retry
|
|
26
|
+
server_error = "server_error" # 500/502: retry
|
|
27
|
+
timeout = "timeout" # connect/read timeout: retry
|
|
28
|
+
context_overflow = "context_overflow" # prompt too long: compress, do not fail over
|
|
29
|
+
payload_too_large = "payload_too_large" # 413, or a 400 naming a body byte cap: compress
|
|
30
|
+
model_not_found = "model_not_found" # 404 / invalid model id: fall back to another model
|
|
31
|
+
provider_policy_blocked = "provider_policy_blocked" # data policy excluded every endpoint
|
|
32
|
+
content_policy_blocked = "content_policy_blocked" # safety filter: same input fails again
|
|
33
|
+
format_error = "format_error" # 400 malformed request: abort
|
|
34
|
+
unknown = "unknown"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class ClassifiedError:
|
|
39
|
+
reason: FailoverReason
|
|
40
|
+
status_code: int | None = None
|
|
41
|
+
message: str = ""
|
|
42
|
+
retryable: bool = False
|
|
43
|
+
should_fallback: bool = False
|
|
44
|
+
should_compress: bool = False
|
|
45
|
+
context: dict[str, Any] = field(default_factory=dict)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ModelAPIError(Exception):
|
|
49
|
+
"""Non-2xx answer from a model endpoint. ``body`` is the parsed JSON when available."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, status_code: int, message: str, body: Any = None):
|
|
52
|
+
super().__init__(f"HTTP {status_code}: {message}")
|
|
53
|
+
self.status_code = status_code
|
|
54
|
+
self.message = message
|
|
55
|
+
self.body = body
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_CONTEXT_PATTERNS = (
|
|
59
|
+
"context length",
|
|
60
|
+
"context_length",
|
|
61
|
+
"maximum context",
|
|
62
|
+
"too many tokens",
|
|
63
|
+
"prompt is too long",
|
|
64
|
+
"reduce the length",
|
|
65
|
+
"context window",
|
|
66
|
+
)
|
|
67
|
+
_BILLING_PATTERNS = ("insufficient credits", "insufficient_quota", "key limit exceeded", "credit")
|
|
68
|
+
_POLICY_PATTERNS = ("data policy", "no endpoints found matching your data policy")
|
|
69
|
+
_CONTENT_PATTERNS = ("content policy", "flagged", "moderation", "safety")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# Hermes v0.23.0: byte-limited gateways report an oversized request BODY as a 400 (Novita
|
|
73
|
+
# "request body too large, must be less than 20MB", Z.AI "Request size exceeds the limit of
|
|
74
|
+
# ..."). Compressing helps; retrying the same body does not. A subject AND an overflow word are
|
|
75
|
+
# both required, so field-length complaints stay out; a cap counted in tokens keeps its rules.
|
|
76
|
+
_BYTE_SIZE_SUBJECT = re.compile(
|
|
77
|
+
r"(?<!response )\b(?:(?:request ?size|(?:request )?(?:body|payload) size)(?: is)? "
|
|
78
|
+
r"(?:exceed|too large|too big)"
|
|
79
|
+
r"|request body(?: is)? too large|(?:request )?payload(?: is)? too large)"
|
|
80
|
+
)
|
|
81
|
+
_BYTE_SIZE_OVERFLOW = re.compile(r"exceed|too large|too big")
|
|
82
|
+
_TOKEN_COUNT = re.compile(r"\btokens?\b")
|
|
83
|
+
_REQUEST_WORDS = {"request", "requestsize", "byte", "bytes"}
|
|
84
|
+
|
|
85
|
+
# Hermes v0.23.0 (timeout_error_summary): say whether a connection was ever made and what to
|
|
86
|
+
# change, instead of the transport's bare "timed out". Both keep the "timed out" wording.
|
|
87
|
+
_CONNECT_TIMEOUT = (
|
|
88
|
+
"Connection to the provider timed out before it was established "
|
|
89
|
+
"(check the base_url and network path)"
|
|
90
|
+
)
|
|
91
|
+
_READ_TIMEOUT = (
|
|
92
|
+
"Request timed out waiting for the provider to respond (if the model needs longer, "
|
|
93
|
+
"raise timeout= on the model client)"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _is_byte_size_limit_message(message: str) -> bool:
|
|
98
|
+
msg = message.lower()
|
|
99
|
+
if not _BYTE_SIZE_OVERFLOW.search(msg) or _TOKEN_COUNT.search(msg):
|
|
100
|
+
return False
|
|
101
|
+
if _BYTE_SIZE_SUBJECT.search(msg):
|
|
102
|
+
return True
|
|
103
|
+
# "content length" alone is a field complaint unless the request or bytes are named. "_"
|
|
104
|
+
# stays inside a word, so the body's "invalid_request_error" type is not "request".
|
|
105
|
+
content_length = "content length" in msg or "content-length" in msg
|
|
106
|
+
return content_length and bool(_REQUEST_WORDS & set(re.split(r"[^a-z0-9_]+", msg)))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _timeout(error: httpx.TimeoutException) -> ClassifiedError:
|
|
110
|
+
# Pool and write timeouts: the provider never got to answer, so neither hint applies.
|
|
111
|
+
if isinstance(error, httpx.ConnectTimeout):
|
|
112
|
+
phase, message = "connect", _CONNECT_TIMEOUT
|
|
113
|
+
elif isinstance(error, httpx.ReadTimeout):
|
|
114
|
+
phase, message = "read", _READ_TIMEOUT
|
|
115
|
+
else:
|
|
116
|
+
phase, message = None, str(error)
|
|
117
|
+
return ClassifiedError(
|
|
118
|
+
FailoverReason.timeout,
|
|
119
|
+
message=message,
|
|
120
|
+
retryable=True,
|
|
121
|
+
should_fallback=True,
|
|
122
|
+
context={"timeout_phase": phase} if phase else {},
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _text(error: ModelAPIError) -> str:
|
|
127
|
+
return f"{error.message} {error.body!r}".lower()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _upstream_429(body: Any) -> bool:
|
|
131
|
+
"""OpenRouter wraps provider errors as ``{"error": {"metadata": {"provider_name": ...}}}``."""
|
|
132
|
+
if not isinstance(body, dict):
|
|
133
|
+
return False
|
|
134
|
+
meta = (
|
|
135
|
+
(body.get("error") or {}).get("metadata") if isinstance(body.get("error"), dict) else None
|
|
136
|
+
)
|
|
137
|
+
return isinstance(meta, dict) and bool(meta.get("provider_name") or meta.get("raw"))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def classify_error(error: BaseException) -> ClassifiedError:
|
|
141
|
+
if isinstance(error, httpx.TimeoutException):
|
|
142
|
+
return _timeout(error)
|
|
143
|
+
if isinstance(error, httpx.TransportError):
|
|
144
|
+
return ClassifiedError(
|
|
145
|
+
FailoverReason.server_error, message=str(error), retryable=True, should_fallback=True
|
|
146
|
+
)
|
|
147
|
+
if not isinstance(error, ModelAPIError):
|
|
148
|
+
return ClassifiedError(FailoverReason.unknown, message=type(error).__name__)
|
|
149
|
+
|
|
150
|
+
status, text = error.status_code, _text(error)
|
|
151
|
+
|
|
152
|
+
def verdict(reason: FailoverReason, **hints: Any) -> ClassifiedError:
|
|
153
|
+
return ClassifiedError(reason, status_code=status, message=error.message, **hints)
|
|
154
|
+
|
|
155
|
+
if any(p in text for p in _CONTEXT_PATTERNS):
|
|
156
|
+
return verdict(FailoverReason.context_overflow, should_compress=True)
|
|
157
|
+
if status == 413:
|
|
158
|
+
return verdict(FailoverReason.payload_too_large, should_compress=True)
|
|
159
|
+
if status == 402 or any(p in text for p in _BILLING_PATTERNS):
|
|
160
|
+
return verdict(FailoverReason.billing, should_fallback=True)
|
|
161
|
+
if status == 401:
|
|
162
|
+
return verdict(FailoverReason.auth)
|
|
163
|
+
if status == 403:
|
|
164
|
+
if any(p in text for p in _POLICY_PATTERNS):
|
|
165
|
+
return verdict(FailoverReason.provider_policy_blocked, should_fallback=True)
|
|
166
|
+
return verdict(FailoverReason.auth)
|
|
167
|
+
if status == 404:
|
|
168
|
+
if any(p in text for p in _POLICY_PATTERNS):
|
|
169
|
+
return verdict(FailoverReason.provider_policy_blocked, should_fallback=True)
|
|
170
|
+
return verdict(FailoverReason.model_not_found, should_fallback=True)
|
|
171
|
+
if status == 429:
|
|
172
|
+
if _upstream_429(error.body):
|
|
173
|
+
return verdict(FailoverReason.upstream_rate_limit, should_fallback=True)
|
|
174
|
+
return verdict(FailoverReason.rate_limit, retryable=True, should_fallback=True)
|
|
175
|
+
if status in (503, 529):
|
|
176
|
+
return verdict(FailoverReason.overloaded, retryable=True, should_fallback=True)
|
|
177
|
+
if 500 <= status < 600:
|
|
178
|
+
return verdict(FailoverReason.server_error, retryable=True, should_fallback=True)
|
|
179
|
+
if status == 400:
|
|
180
|
+
if _is_byte_size_limit_message(error.message):
|
|
181
|
+
return verdict(FailoverReason.payload_too_large, should_compress=True)
|
|
182
|
+
if any(p in text for p in _CONTENT_PATTERNS):
|
|
183
|
+
return verdict(FailoverReason.content_policy_blocked)
|
|
184
|
+
return verdict(FailoverReason.format_error)
|
|
185
|
+
return verdict(FailoverReason.unknown)
|
cairnkit/llm/fallback.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Adapted from hermes-agent v0.23.0 (2bd378e): agent/fallback_cooldown.py, the fallback walk
|
|
2
|
+
# in agent/chat_completion_helpers.py and agent/timeout_error_summary.py.
|
|
3
|
+
# Copyright (c) 2025 Nous Research. MIT License.
|
|
4
|
+
"""Try a chain of models; retry transient errors, fall back on model-level failures."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import logging
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Awaitable, Callable
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from cairnkit.llm.base import ChatModel, emit_delta
|
|
15
|
+
from cairnkit.llm.errors import ClassifiedError, FailoverReason, classify_error
|
|
16
|
+
from cairnkit.types import Message, ModelResponse
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
Sleep = Callable[[float], Awaitable[None]]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _label(err: ClassifiedError) -> str:
|
|
24
|
+
"""The failure class, plus which phase timed out (Hermes v0.23.0 timeout_error_summary)."""
|
|
25
|
+
if err.context.get("timeout_phase"):
|
|
26
|
+
return f"{err.reason.value} ({err.message})"
|
|
27
|
+
return err.reason.value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AllModelsFailed(Exception):
|
|
31
|
+
def __init__(self, errors: list[tuple[str, ClassifiedError]]):
|
|
32
|
+
summary = "; ".join(f"{name}: {_label(err)}" for name, err in errors)
|
|
33
|
+
super().__init__(f"every model failed ({summary})")
|
|
34
|
+
self.errors = errors
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class FallbackModel:
|
|
38
|
+
"""Primary model first; on fallback-worthy failures move down the chain.
|
|
39
|
+
|
|
40
|
+
A model that hit a rate limit is skipped for ``cooldown_seconds`` so later calls
|
|
41
|
+
go straight to the next model instead of waiting on a throttled one.
|
|
42
|
+
Errors that no other model can fix (auth, malformed request, context overflow)
|
|
43
|
+
are raised immediately.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
models: list[ChatModel],
|
|
49
|
+
*,
|
|
50
|
+
retries: int = 2,
|
|
51
|
+
backoff_seconds: float = 1.0,
|
|
52
|
+
cooldown_seconds: float = 60.0,
|
|
53
|
+
sleep: Sleep = asyncio.sleep,
|
|
54
|
+
clock: Callable[[], float] = time.monotonic,
|
|
55
|
+
):
|
|
56
|
+
if not models:
|
|
57
|
+
raise ValueError("at least one model is required")
|
|
58
|
+
self.models = models
|
|
59
|
+
self.name = models[0].name
|
|
60
|
+
self._retries = retries
|
|
61
|
+
self._backoff = backoff_seconds
|
|
62
|
+
self._cooldown = cooldown_seconds
|
|
63
|
+
self._sleep = sleep
|
|
64
|
+
self._clock = clock
|
|
65
|
+
self._cooling_until: dict[str, float] = {}
|
|
66
|
+
|
|
67
|
+
async def complete(self, messages: list[Message], **kwargs: Any) -> ModelResponse:
|
|
68
|
+
"""Streamed text cannot be taken back, so once a delta went out, errors are raised."""
|
|
69
|
+
errors: list[tuple[str, ClassifiedError]] = []
|
|
70
|
+
user_delta = kwargs.pop("on_delta", None)
|
|
71
|
+
streamed = False
|
|
72
|
+
|
|
73
|
+
async def on_delta(text: str) -> None:
|
|
74
|
+
nonlocal streamed
|
|
75
|
+
streamed = True
|
|
76
|
+
await emit_delta(user_delta, text)
|
|
77
|
+
|
|
78
|
+
if user_delta is not None:
|
|
79
|
+
kwargs["on_delta"] = on_delta
|
|
80
|
+
now = self._clock()
|
|
81
|
+
candidates = [m for m in self.models if self._cooling_until.get(m.name, 0) <= now]
|
|
82
|
+
for model in candidates or self.models:
|
|
83
|
+
for attempt in range(self._retries + 1):
|
|
84
|
+
try:
|
|
85
|
+
return await model.complete(messages, **kwargs)
|
|
86
|
+
except Exception as exc: # noqa: BLE001 - classified below
|
|
87
|
+
if streamed:
|
|
88
|
+
raise
|
|
89
|
+
verdict = classify_error(exc)
|
|
90
|
+
logger.warning("model %s failed: %s", model.name, verdict.reason.value)
|
|
91
|
+
if verdict.reason in (
|
|
92
|
+
FailoverReason.rate_limit,
|
|
93
|
+
FailoverReason.upstream_rate_limit,
|
|
94
|
+
):
|
|
95
|
+
self._cooling_until[model.name] = self._clock() + self._cooldown
|
|
96
|
+
if not verdict.retryable and not verdict.should_fallback:
|
|
97
|
+
raise
|
|
98
|
+
if (
|
|
99
|
+
verdict.retryable
|
|
100
|
+
and attempt < self._retries
|
|
101
|
+
and verdict.reason not in (FailoverReason.rate_limit,)
|
|
102
|
+
):
|
|
103
|
+
await self._sleep(self._backoff * (2**attempt))
|
|
104
|
+
continue
|
|
105
|
+
errors.append((model.name, verdict))
|
|
106
|
+
break
|
|
107
|
+
raise AllModelsFailed(errors)
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
# Adapted from hermes-agent v0.23.0 (2bd378e): agent/chat_completion_helpers_tool_calls.py
|
|
2
|
+
# (streamed tool-call assembly). Copyright (c) 2025 Nous Research. MIT License.
|
|
3
|
+
"""Client for any OpenAI-compatible ``/chat/completions`` endpoint."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from cairnkit.llm.base import DeltaHandler, emit_delta
|
|
13
|
+
from cairnkit.llm.errors import ModelAPIError
|
|
14
|
+
from cairnkit.types import Message, ModelResponse, ToolCall, Usage
|
|
15
|
+
|
|
16
|
+
OPENROUTER_URL = "https://openrouter.ai/api/v1"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class OpenAICompatModel:
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
model: str,
|
|
23
|
+
*,
|
|
24
|
+
base_url: str,
|
|
25
|
+
api_key: str,
|
|
26
|
+
extra_body: dict[str, Any] | None = None,
|
|
27
|
+
headers: dict[str, str] | None = None,
|
|
28
|
+
timeout: float = 60.0,
|
|
29
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
30
|
+
):
|
|
31
|
+
if not api_key:
|
|
32
|
+
raise ValueError("api_key is required")
|
|
33
|
+
self.name = model
|
|
34
|
+
self._url = base_url.rstrip("/") + "/chat/completions"
|
|
35
|
+
self._headers = {"Authorization": f"Bearer {api_key}", **(headers or {})}
|
|
36
|
+
self._extra_body = extra_body or {}
|
|
37
|
+
self._timeout = timeout
|
|
38
|
+
self._transport = transport
|
|
39
|
+
|
|
40
|
+
async def complete(
|
|
41
|
+
self,
|
|
42
|
+
messages: list[Message],
|
|
43
|
+
*,
|
|
44
|
+
tools: list[dict[str, Any]] | None = None,
|
|
45
|
+
temperature: float | None = None,
|
|
46
|
+
max_tokens: int | None = None,
|
|
47
|
+
on_delta: DeltaHandler | None = None,
|
|
48
|
+
) -> ModelResponse:
|
|
49
|
+
body: dict[str, Any] = {"model": self.name, "messages": messages, **self._extra_body}
|
|
50
|
+
if tools:
|
|
51
|
+
body["tools"] = tools
|
|
52
|
+
if temperature is not None:
|
|
53
|
+
body["temperature"] = temperature
|
|
54
|
+
if max_tokens is not None:
|
|
55
|
+
body["max_tokens"] = max_tokens
|
|
56
|
+
if on_delta is not None:
|
|
57
|
+
return await self._stream(body, on_delta)
|
|
58
|
+
async with httpx.AsyncClient(timeout=self._timeout, transport=self._transport) as client:
|
|
59
|
+
response = await client.post(self._url, json=body, headers=self._headers)
|
|
60
|
+
payload = _json(response)
|
|
61
|
+
if response.status_code >= 400:
|
|
62
|
+
raise ModelAPIError(response.status_code, _error_message(payload, response), payload)
|
|
63
|
+
# Some gateways answer 200 with an error object instead of choices.
|
|
64
|
+
if isinstance(payload, dict) and payload.get("error") and not payload.get("choices"):
|
|
65
|
+
error = payload["error"]
|
|
66
|
+
code = error.get("code") if isinstance(error, dict) else None
|
|
67
|
+
raise ModelAPIError(
|
|
68
|
+
code if isinstance(code, int) else 502, _error_message(payload, response), payload
|
|
69
|
+
)
|
|
70
|
+
return _parse(payload, self.name)
|
|
71
|
+
|
|
72
|
+
async def _stream(self, body: dict[str, Any], on_delta: DeltaHandler) -> ModelResponse:
|
|
73
|
+
body = {**body, "stream": True, "stream_options": {"include_usage": True}}
|
|
74
|
+
content: list[str] = []
|
|
75
|
+
calls = _ToolCallAccumulator()
|
|
76
|
+
final: dict[str, Any] = {"model": self.name, "finish_reason": None, "usage": {}}
|
|
77
|
+
async with (
|
|
78
|
+
httpx.AsyncClient(timeout=self._timeout, transport=self._transport) as client,
|
|
79
|
+
client.stream("POST", self._url, json=body, headers=self._headers) as response,
|
|
80
|
+
):
|
|
81
|
+
if response.status_code >= 400:
|
|
82
|
+
await response.aread()
|
|
83
|
+
payload = _json(response)
|
|
84
|
+
raise ModelAPIError(
|
|
85
|
+
response.status_code, _error_message(payload, response), payload
|
|
86
|
+
)
|
|
87
|
+
async for line in response.aiter_lines():
|
|
88
|
+
if not line.startswith("data:"):
|
|
89
|
+
continue # SSE comments (": OPENROUTER PROCESSING") and blank lines
|
|
90
|
+
data = line[5:].strip()
|
|
91
|
+
if data == "[DONE]":
|
|
92
|
+
break
|
|
93
|
+
try:
|
|
94
|
+
chunk = json.loads(data)
|
|
95
|
+
except ValueError:
|
|
96
|
+
continue
|
|
97
|
+
if isinstance(chunk.get("error"), dict):
|
|
98
|
+
error = chunk["error"]
|
|
99
|
+
code = error.get("code") if isinstance(error.get("code"), int) else 502
|
|
100
|
+
raise ModelAPIError(code, str(error.get("message", "stream error")), chunk)
|
|
101
|
+
final["model"] = chunk.get("model") or final["model"]
|
|
102
|
+
if chunk.get("usage"):
|
|
103
|
+
final["usage"] = chunk["usage"]
|
|
104
|
+
for choice in chunk.get("choices") or []:
|
|
105
|
+
delta = choice.get("delta") or {}
|
|
106
|
+
if text := delta.get("content"):
|
|
107
|
+
content.append(text)
|
|
108
|
+
await emit_delta(on_delta, text)
|
|
109
|
+
for raw in delta.get("tool_calls") or []:
|
|
110
|
+
calls.feed(raw)
|
|
111
|
+
final["finish_reason"] = choice.get("finish_reason") or final["finish_reason"]
|
|
112
|
+
message = {
|
|
113
|
+
"content": "".join(content) or None,
|
|
114
|
+
"tool_calls": [
|
|
115
|
+
{
|
|
116
|
+
"id": c["id"] or f"call_{i}",
|
|
117
|
+
"function": {"name": c["name"], "arguments": "".join(c["args"])},
|
|
118
|
+
}
|
|
119
|
+
for i, c in sorted(calls.slots.items())
|
|
120
|
+
],
|
|
121
|
+
}
|
|
122
|
+
return _parse(
|
|
123
|
+
{
|
|
124
|
+
"model": final["model"],
|
|
125
|
+
"usage": final["usage"],
|
|
126
|
+
"choices": [{"message": message, "finish_reason": final["finish_reason"]}],
|
|
127
|
+
},
|
|
128
|
+
self.name,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _ToolCallAccumulator:
|
|
133
|
+
"""Assemble streamed tool-call deltas into whole calls (slot -> id, name, argument parts).
|
|
134
|
+
|
|
135
|
+
Ollama-compatible endpoints reuse index 0 for every call of a parallel batch, told apart
|
|
136
|
+
only by id, so a new id at a seen index can open a new slot. LM Studio and Kimi send a fresh
|
|
137
|
+
id on every argument fragment without a name, and some MiniMax routes resend the same name
|
|
138
|
+
with each fragment: only a named delta opens a call, and one with the current call's name
|
|
139
|
+
only after that call's arguments have closed.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
def __init__(self) -> None:
|
|
143
|
+
self.slots: dict[int, dict[str, Any]] = {}
|
|
144
|
+
self._slot_by_index: dict[int, int] = {}
|
|
145
|
+
self._last_id: dict[int, str] = {}
|
|
146
|
+
# slot -> (fragments scanned, depth, in_string, escaped, started)
|
|
147
|
+
self._scan: dict[int, tuple[int, int, bool, bool, bool]] = {}
|
|
148
|
+
|
|
149
|
+
def feed(self, raw: dict[str, Any]) -> None:
|
|
150
|
+
index = raw.get("index")
|
|
151
|
+
index = index if isinstance(index, int) else 0
|
|
152
|
+
call_id = str(raw.get("id") or "")
|
|
153
|
+
function = raw.get("function") or {}
|
|
154
|
+
slot = self._slot_by_index.setdefault(index, index)
|
|
155
|
+
last = self._last_id.get(index)
|
|
156
|
+
if (
|
|
157
|
+
call_id
|
|
158
|
+
and last is not None
|
|
159
|
+
and call_id != last
|
|
160
|
+
and self._starts_new_call(function, slot)
|
|
161
|
+
):
|
|
162
|
+
slot = self._slot_by_index[index] = max(self.slots, default=-1) + 1
|
|
163
|
+
if call_id:
|
|
164
|
+
self._last_id[index] = call_id
|
|
165
|
+
entry = self.slots.setdefault(slot, {"id": "", "name": "", "args": []})
|
|
166
|
+
entry["id"] = call_id or entry["id"]
|
|
167
|
+
# Assignment, not +=: some providers resend the full name with every chunk.
|
|
168
|
+
entry["name"] = function.get("name") or entry["name"]
|
|
169
|
+
if function.get("arguments"):
|
|
170
|
+
entry["args"].append(function["arguments"])
|
|
171
|
+
|
|
172
|
+
def _starts_new_call(self, function: dict[str, Any], slot: int) -> bool:
|
|
173
|
+
name = function.get("name")
|
|
174
|
+
if not name:
|
|
175
|
+
return False
|
|
176
|
+
if name != self.slots.get(slot, {}).get("name"):
|
|
177
|
+
return True
|
|
178
|
+
return self._arguments_closed(slot)
|
|
179
|
+
|
|
180
|
+
def _arguments_closed(self, slot: int) -> bool:
|
|
181
|
+
"""Brackets balanced outside strings (not strict JSON, so a repairable call still ends
|
|
182
|
+
its slot). Scans only fragments added since the last check."""
|
|
183
|
+
parts = self.slots.get(slot, {}).get("args", [])
|
|
184
|
+
done, depth, in_string, escaped, started = self._scan.get(slot, (0, 0, False, False, False))
|
|
185
|
+
for ch in "".join(parts[done:]):
|
|
186
|
+
if in_string:
|
|
187
|
+
if escaped:
|
|
188
|
+
escaped = False
|
|
189
|
+
elif ch == "\\":
|
|
190
|
+
escaped = True
|
|
191
|
+
elif ch == '"':
|
|
192
|
+
in_string = False
|
|
193
|
+
elif ch == '"':
|
|
194
|
+
in_string = True
|
|
195
|
+
elif ch in "{[":
|
|
196
|
+
depth, started = depth + 1, True
|
|
197
|
+
elif ch in "}]":
|
|
198
|
+
depth -= 1
|
|
199
|
+
self._scan[slot] = (len(parts), depth, in_string, escaped, started)
|
|
200
|
+
return started and depth == 0 and not in_string
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def openrouter(
|
|
204
|
+
model: str,
|
|
205
|
+
*,
|
|
206
|
+
api_key: str,
|
|
207
|
+
deny_data_collection: bool = True,
|
|
208
|
+
sort: str | None = "throughput",
|
|
209
|
+
only_providers: list[str] | None = None,
|
|
210
|
+
app_name: str | None = None,
|
|
211
|
+
timeout: float = 60.0,
|
|
212
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
213
|
+
) -> OpenAICompatModel:
|
|
214
|
+
"""OpenRouter with privacy-first routing: providers that store/train on data are excluded."""
|
|
215
|
+
provider: dict[str, Any] = {}
|
|
216
|
+
if deny_data_collection:
|
|
217
|
+
provider["data_collection"] = "deny"
|
|
218
|
+
if sort:
|
|
219
|
+
provider["sort"] = sort
|
|
220
|
+
if only_providers:
|
|
221
|
+
provider["only"] = only_providers
|
|
222
|
+
headers = {"X-Title": app_name} if app_name else None
|
|
223
|
+
return OpenAICompatModel(
|
|
224
|
+
model,
|
|
225
|
+
base_url=OPENROUTER_URL,
|
|
226
|
+
api_key=api_key,
|
|
227
|
+
extra_body={"provider": provider, "usage": {"include": True}} if provider else None,
|
|
228
|
+
headers=headers,
|
|
229
|
+
timeout=timeout,
|
|
230
|
+
transport=transport,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _json(response: httpx.Response) -> Any:
|
|
235
|
+
try:
|
|
236
|
+
return response.json()
|
|
237
|
+
except ValueError:
|
|
238
|
+
return None
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _error_message(payload: Any, response: httpx.Response) -> str:
|
|
242
|
+
if isinstance(payload, dict) and isinstance(payload.get("error"), dict):
|
|
243
|
+
return str(payload["error"].get("message") or response.reason_phrase)
|
|
244
|
+
return response.reason_phrase or "model request failed"
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _parse(payload: Any, fallback_model: str) -> ModelResponse:
|
|
248
|
+
try:
|
|
249
|
+
choice = payload["choices"][0]
|
|
250
|
+
message = choice["message"]
|
|
251
|
+
except (KeyError, IndexError, TypeError) as exc:
|
|
252
|
+
raise ModelAPIError(502, "malformed completion payload", payload) from exc
|
|
253
|
+
calls = []
|
|
254
|
+
for raw in message.get("tool_calls") or []:
|
|
255
|
+
function = raw.get("function") or {}
|
|
256
|
+
arguments = function.get("arguments")
|
|
257
|
+
if not isinstance(arguments, str):
|
|
258
|
+
arguments = json.dumps(arguments or {})
|
|
259
|
+
calls.append(
|
|
260
|
+
ToolCall(
|
|
261
|
+
id=raw.get("id") or f"call_{len(calls)}",
|
|
262
|
+
name=function.get("name", ""),
|
|
263
|
+
arguments=arguments,
|
|
264
|
+
)
|
|
265
|
+
)
|
|
266
|
+
usage_raw = payload.get("usage") or {}
|
|
267
|
+
details = usage_raw.get("prompt_tokens_details") or {}
|
|
268
|
+
usage = Usage(
|
|
269
|
+
prompt_tokens=int(usage_raw.get("prompt_tokens") or 0),
|
|
270
|
+
completion_tokens=int(usage_raw.get("completion_tokens") or 0),
|
|
271
|
+
cached_tokens=int(details.get("cached_tokens") or 0),
|
|
272
|
+
cost=float(usage_raw["cost"]) if isinstance(usage_raw.get("cost"), int | float) else None,
|
|
273
|
+
calls=1,
|
|
274
|
+
)
|
|
275
|
+
return ModelResponse(
|
|
276
|
+
content=message.get("content"),
|
|
277
|
+
tool_calls=calls,
|
|
278
|
+
usage=usage,
|
|
279
|
+
model=payload.get("model") or fallback_model,
|
|
280
|
+
finish_reason=choice.get("finish_reason"),
|
|
281
|
+
)
|
cairnkit/llm/scripted.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Deterministic model for tests, demos and offline development."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from cairnkit.llm.base import DeltaHandler, emit_delta
|
|
9
|
+
from cairnkit.types import Message, ModelResponse, Usage
|
|
10
|
+
|
|
11
|
+
Responder = Callable[[list[Message], list[dict[str, Any]] | None], ModelResponse]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ScriptedModel:
|
|
15
|
+
"""Answers from a list of canned responses, or by calling ``responder``."""
|
|
16
|
+
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
responses: list[ModelResponse] | None = None,
|
|
20
|
+
*,
|
|
21
|
+
responder: Responder | None = None,
|
|
22
|
+
name: str = "scripted",
|
|
23
|
+
):
|
|
24
|
+
if (responses is None) == (responder is None):
|
|
25
|
+
raise ValueError("pass exactly one of responses or responder")
|
|
26
|
+
self.name = name
|
|
27
|
+
self._responses = list(responses or [])
|
|
28
|
+
self._responder = responder
|
|
29
|
+
self.requests: list[dict[str, Any]] = []
|
|
30
|
+
|
|
31
|
+
async def complete(
|
|
32
|
+
self,
|
|
33
|
+
messages: list[Message],
|
|
34
|
+
*,
|
|
35
|
+
tools: list[dict[str, Any]] | None = None,
|
|
36
|
+
temperature: float | None = None,
|
|
37
|
+
max_tokens: int | None = None,
|
|
38
|
+
on_delta: DeltaHandler | None = None,
|
|
39
|
+
) -> ModelResponse:
|
|
40
|
+
self.requests.append({"messages": [dict(m) for m in messages], "tools": tools})
|
|
41
|
+
if self._responder is not None:
|
|
42
|
+
response = self._responder(messages, tools)
|
|
43
|
+
else:
|
|
44
|
+
if not self._responses:
|
|
45
|
+
raise RuntimeError("ScriptedModel ran out of responses")
|
|
46
|
+
response = self._responses.pop(0)
|
|
47
|
+
if response.usage.calls == 0:
|
|
48
|
+
response.usage = Usage(calls=1)
|
|
49
|
+
response.model = response.model or self.name
|
|
50
|
+
if on_delta is not None and response.content:
|
|
51
|
+
words = response.content.split(" ")
|
|
52
|
+
for i, word in enumerate(words):
|
|
53
|
+
await emit_delta(on_delta, word if i == len(words) - 1 else word + " ")
|
|
54
|
+
return response
|