inferenceswitch 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inferenceswitch/__init__.py +104 -0
- inferenceswitch/adapters/__init__.py +34 -0
- inferenceswitch/adapters/anthropic.py +433 -0
- inferenceswitch/adapters/base.py +152 -0
- inferenceswitch/adapters/gemini.py +480 -0
- inferenceswitch/adapters/openai_compat.py +283 -0
- inferenceswitch/capabilities.py +98 -0
- inferenceswitch/client.py +527 -0
- inferenceswitch/errors.py +57 -0
- inferenceswitch/messages.py +212 -0
- inferenceswitch/registry.py +296 -0
- inferenceswitch/schema.py +162 -0
- inferenceswitch-0.1.0.dist-info/METADATA +476 -0
- inferenceswitch-0.1.0.dist-info/RECORD +16 -0
- inferenceswitch-0.1.0.dist-info/WHEEL +4 -0
- inferenceswitch-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""inferenceswitch — lightweight, import-only multi-provider LLM dispatch.
|
|
2
|
+
|
|
3
|
+
from inferenceswitch import Client
|
|
4
|
+
|
|
5
|
+
client = Client()
|
|
6
|
+
data = client.generate_structured_json(
|
|
7
|
+
model="claude-opus-4-8", # or "gemini-3.5-flash", "groq/llama-3.3-70b", ...
|
|
8
|
+
schema=MyModel.model_json_schema(),
|
|
9
|
+
prompt="...",
|
|
10
|
+
system="...",
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
No proxy, no server, no dynamic provider loading — just a static registry and a
|
|
14
|
+
thick client that handles per-provider structured-output mechanics for you.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from .adapters.anthropic import (
|
|
19
|
+
CLAUDE_MAX_OUTPUT_TOKENS,
|
|
20
|
+
UNKNOWN_MODEL_MAX_OUTPUT_TOKENS,
|
|
21
|
+
claude_max_output_tokens,
|
|
22
|
+
)
|
|
23
|
+
from .adapters.gemini import (
|
|
24
|
+
GEMINI_MAX_OUTPUT_TOKENS,
|
|
25
|
+
UNKNOWN_GEMINI_MODEL_MAX_OUTPUT_TOKENS,
|
|
26
|
+
gemini_max_output_tokens,
|
|
27
|
+
)
|
|
28
|
+
from .capabilities import Capabilities, Capability, SchemaDialect, StructuredMode
|
|
29
|
+
from .client import Client, ModelChoice, Resolution, env_key_provider
|
|
30
|
+
from .messages import (
|
|
31
|
+
CacheHandle,
|
|
32
|
+
Effort,
|
|
33
|
+
Response,
|
|
34
|
+
Message,
|
|
35
|
+
StopReason,
|
|
36
|
+
Text,
|
|
37
|
+
Tool,
|
|
38
|
+
ToolCall,
|
|
39
|
+
ToolChoice,
|
|
40
|
+
ToolResult,
|
|
41
|
+
ToolUse,
|
|
42
|
+
Usage,
|
|
43
|
+
assistant,
|
|
44
|
+
system,
|
|
45
|
+
tool_result,
|
|
46
|
+
user,
|
|
47
|
+
)
|
|
48
|
+
from .errors import (
|
|
49
|
+
InferenceSwitchError,
|
|
50
|
+
MissingAPIKeyError,
|
|
51
|
+
MissingDependencyError,
|
|
52
|
+
ProviderResolutionError,
|
|
53
|
+
StructuredOutputError,
|
|
54
|
+
UnsupportedCapabilityError,
|
|
55
|
+
)
|
|
56
|
+
from .registry import ProviderSpec, Registry, default_registry
|
|
57
|
+
|
|
58
|
+
__version__ = "0.1.0"
|
|
59
|
+
|
|
60
|
+
__all__ = [
|
|
61
|
+
"Client",
|
|
62
|
+
"ModelChoice",
|
|
63
|
+
"Resolution",
|
|
64
|
+
"env_key_provider",
|
|
65
|
+
"Registry",
|
|
66
|
+
"ProviderSpec",
|
|
67
|
+
"default_registry",
|
|
68
|
+
"Capabilities",
|
|
69
|
+
"Capability",
|
|
70
|
+
"StructuredMode",
|
|
71
|
+
"SchemaDialect",
|
|
72
|
+
"InferenceSwitchError",
|
|
73
|
+
"ProviderResolutionError",
|
|
74
|
+
"MissingAPIKeyError",
|
|
75
|
+
"MissingDependencyError",
|
|
76
|
+
"StructuredOutputError",
|
|
77
|
+
"UnsupportedCapabilityError",
|
|
78
|
+
# chat / tool-calling surface
|
|
79
|
+
"Message",
|
|
80
|
+
"Tool",
|
|
81
|
+
"ToolCall",
|
|
82
|
+
"ToolChoice",
|
|
83
|
+
"ToolUse",
|
|
84
|
+
"ToolResult",
|
|
85
|
+
"Text",
|
|
86
|
+
"CacheHandle",
|
|
87
|
+
"Effort",
|
|
88
|
+
"Response",
|
|
89
|
+
"StopReason",
|
|
90
|
+
"Usage",
|
|
91
|
+
# Per-provider output caps (default = the model's own maximum). Editing a
|
|
92
|
+
# table imposes a house ceiling process-wide; a per-call max_tokens still wins.
|
|
93
|
+
"CLAUDE_MAX_OUTPUT_TOKENS",
|
|
94
|
+
"UNKNOWN_MODEL_MAX_OUTPUT_TOKENS",
|
|
95
|
+
"claude_max_output_tokens",
|
|
96
|
+
"GEMINI_MAX_OUTPUT_TOKENS",
|
|
97
|
+
"UNKNOWN_GEMINI_MODEL_MAX_OUTPUT_TOKENS",
|
|
98
|
+
"gemini_max_output_tokens",
|
|
99
|
+
"user",
|
|
100
|
+
"assistant",
|
|
101
|
+
"system",
|
|
102
|
+
"tool_result",
|
|
103
|
+
"__version__",
|
|
104
|
+
]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Adapter registry: maps a provider ``kind`` to its adapter class."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from ..registry import KIND_ANTHROPIC, KIND_GEMINI, KIND_OPENAI, ProviderSpec
|
|
5
|
+
from .anthropic import AnthropicAdapter
|
|
6
|
+
from .base import Adapter
|
|
7
|
+
from .gemini import GeminiAdapter
|
|
8
|
+
from .openai_compat import OpenAICompatibleAdapter
|
|
9
|
+
|
|
10
|
+
_ADAPTERS_BY_KIND: dict[str, type[Adapter]] = {
|
|
11
|
+
KIND_OPENAI: OpenAICompatibleAdapter,
|
|
12
|
+
KIND_ANTHROPIC: AnthropicAdapter,
|
|
13
|
+
KIND_GEMINI: GeminiAdapter,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_adapter(spec: ProviderSpec, api_key: str, *, raw_client=None) -> Adapter:
|
|
18
|
+
try:
|
|
19
|
+
cls = _ADAPTERS_BY_KIND[spec.kind]
|
|
20
|
+
except KeyError:
|
|
21
|
+
raise ValueError(
|
|
22
|
+
f"No adapter registered for provider kind {spec.kind!r} "
|
|
23
|
+
f"(provider {spec.name!r})."
|
|
24
|
+
) from None
|
|
25
|
+
return cls(spec, api_key, raw_client=raw_client)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"Adapter",
|
|
30
|
+
"AnthropicAdapter",
|
|
31
|
+
"GeminiAdapter",
|
|
32
|
+
"OpenAICompatibleAdapter",
|
|
33
|
+
"build_adapter",
|
|
34
|
+
]
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
"""Native Anthropic adapter.
|
|
2
|
+
|
|
3
|
+
Anthropic is not OpenAI-shaped: schema-constrained JSON is obtained by declaring
|
|
4
|
+
the schema as a tool's ``input_schema`` and forcing that tool via ``tool_choice``,
|
|
5
|
+
then reading ``tool_use.input``. Anthropic's schema dialect is the most permissive
|
|
6
|
+
of the three, so no translation is needed on the way in or out.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from ..errors import StructuredOutputError
|
|
13
|
+
from ..messages import (
|
|
14
|
+
CacheHandle,
|
|
15
|
+
Effort,
|
|
16
|
+
Response,
|
|
17
|
+
Message,
|
|
18
|
+
StopReason,
|
|
19
|
+
Text,
|
|
20
|
+
Tool,
|
|
21
|
+
ToolCall,
|
|
22
|
+
ToolChoice,
|
|
23
|
+
ToolResult,
|
|
24
|
+
ToolUse,
|
|
25
|
+
Usage,
|
|
26
|
+
)
|
|
27
|
+
from .base import Adapter, require_sdk, split_system
|
|
28
|
+
|
|
29
|
+
#: Per-model output ceilings for the Claude line. Anthropic requires ``max_tokens``
|
|
30
|
+
#: on every request, so inferenceswitch has to pick a default when the caller doesn't —
|
|
31
|
+
#: and the right default is *that model's* maximum: anything lower silently
|
|
32
|
+
#: truncates a long answer (``stop_reason: "max_tokens"``) for no benefit, since
|
|
33
|
+
#: output is billed per token generated, not per token requested.
|
|
34
|
+
#:
|
|
35
|
+
#: This table is the control surface. It is deliberately a plain mutable dict:
|
|
36
|
+
#: add a row for a model inferenceswitch doesn't know yet, or lower one to put a hard
|
|
37
|
+
#: ceiling on a model process-wide::
|
|
38
|
+
#:
|
|
39
|
+
#: from inferenceswitch import CLAUDE_MAX_OUTPUT_TOKENS
|
|
40
|
+
#: CLAUDE_MAX_OUTPUT_TOKENS["claude-opus-5"] = 32_000 # house limit
|
|
41
|
+
#:
|
|
42
|
+
#: A per-call ``max_tokens=`` still wins over the table, and is never clamped to
|
|
43
|
+
#: it — an explicit number is the caller's decision, and an out-of-range one
|
|
44
|
+
#: surfaces Anthropic's own 400 rather than being silently rewritten.
|
|
45
|
+
CLAUDE_MAX_OUTPUT_TOKENS: dict[str, int] = {
|
|
46
|
+
"claude-fable-5": 128_000,
|
|
47
|
+
"claude-mythos-5": 128_000,
|
|
48
|
+
"claude-opus-5": 128_000,
|
|
49
|
+
"claude-opus-4-8": 128_000,
|
|
50
|
+
"claude-opus-4-7": 128_000,
|
|
51
|
+
"claude-opus-4-6": 128_000,
|
|
52
|
+
"claude-sonnet-5": 128_000,
|
|
53
|
+
"claude-sonnet-4-6": 128_000,
|
|
54
|
+
"claude-haiku-4-5": 64_000,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
#: Used when a model ID matches nothing in the table. 64000 is the smallest
|
|
58
|
+
#: ceiling in the *current* Claude line (Haiku 4.5), so it is the largest value
|
|
59
|
+
#: that is safe to send blind to any current model. Older models cap lower — add
|
|
60
|
+
#: an explicit row above before using one.
|
|
61
|
+
UNKNOWN_MODEL_MAX_OUTPUT_TOKENS = 64_000
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def claude_max_output_tokens(model: str) -> int:
|
|
65
|
+
"""This model's maximum output tokens, per :data:`CLAUDE_MAX_OUTPUT_TOKENS`.
|
|
66
|
+
|
|
67
|
+
Falls back to substring matching on the longest known ID so dated snapshots
|
|
68
|
+
(``claude-haiku-4-5-20251001``) and provider-prefixed IDs
|
|
69
|
+
(``anthropic.claude-opus-4-8`` on Bedrock) resolve to the same ceiling as the
|
|
70
|
+
bare alias, then to :data:`UNKNOWN_MODEL_MAX_OUTPUT_TOKENS`.
|
|
71
|
+
"""
|
|
72
|
+
exact = CLAUDE_MAX_OUTPUT_TOKENS.get(model)
|
|
73
|
+
if exact is not None:
|
|
74
|
+
return exact
|
|
75
|
+
matches = [name for name in CLAUDE_MAX_OUTPUT_TOKENS if name in model]
|
|
76
|
+
if matches:
|
|
77
|
+
return CLAUDE_MAX_OUTPUT_TOKENS[max(matches, key=len)]
|
|
78
|
+
return UNKNOWN_MODEL_MAX_OUTPUT_TOKENS
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def resolve_max_tokens(model: str, max_tokens: int | None) -> int:
|
|
82
|
+
"""The caller's ``max_tokens`` if given, else the model's own maximum."""
|
|
83
|
+
return max_tokens if max_tokens is not None else claude_max_output_tokens(model)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
_ANTHROPIC_EFFORT = {
|
|
87
|
+
Effort.LOW: "low",
|
|
88
|
+
Effort.MEDIUM: "medium",
|
|
89
|
+
Effort.HIGH: "high",
|
|
90
|
+
Effort.MAX: "max",
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def anthropic_reasoning_kwargs(effort: Effort | None) -> dict:
|
|
95
|
+
"""Reasoning via adaptive thinking + ``output_config.effort`` on current
|
|
96
|
+
Anthropic models. ``NONE`` disables thinking. (Effort support is per-model —
|
|
97
|
+
e.g. Haiku 4.5 doesn't take it — so a request with effort on a model that
|
|
98
|
+
lacks it will surface the provider's own 400.)"""
|
|
99
|
+
if effort is None:
|
|
100
|
+
return {}
|
|
101
|
+
if effort is Effort.NONE:
|
|
102
|
+
return {"thinking": {"type": "disabled"}}
|
|
103
|
+
return {
|
|
104
|
+
"thinking": {"type": "adaptive"},
|
|
105
|
+
"output_config": {"effort": _ANTHROPIC_EFFORT[effort]},
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
_STOP_REASON = {
|
|
109
|
+
"end_turn": StopReason.END_TURN,
|
|
110
|
+
"max_tokens": StopReason.MAX_TOKENS,
|
|
111
|
+
"tool_use": StopReason.TOOL_USE,
|
|
112
|
+
"stop_sequence": StopReason.STOP_SEQUENCE,
|
|
113
|
+
"refusal": StopReason.REFUSAL,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def encode_anthropic_messages(messages: list[Message]) -> list[dict]:
|
|
118
|
+
"""Normalized messages -> Anthropic content-block messages."""
|
|
119
|
+
out: list[dict] = []
|
|
120
|
+
for message in messages:
|
|
121
|
+
blocks: list[dict] = []
|
|
122
|
+
for part in message.parts():
|
|
123
|
+
if isinstance(part, Text):
|
|
124
|
+
if part.text:
|
|
125
|
+
block = {"type": "text", "text": part.text}
|
|
126
|
+
if part.cache:
|
|
127
|
+
block["cache_control"] = {"type": "ephemeral"}
|
|
128
|
+
blocks.append(block)
|
|
129
|
+
elif isinstance(part, ToolUse):
|
|
130
|
+
blocks.append(
|
|
131
|
+
{"type": "tool_use", "id": part.id, "name": part.name, "input": part.input}
|
|
132
|
+
)
|
|
133
|
+
elif isinstance(part, ToolResult):
|
|
134
|
+
blocks.append(
|
|
135
|
+
{
|
|
136
|
+
"type": "tool_result",
|
|
137
|
+
"tool_use_id": part.tool_call_id,
|
|
138
|
+
"content": part.content,
|
|
139
|
+
"is_error": part.is_error,
|
|
140
|
+
}
|
|
141
|
+
)
|
|
142
|
+
out.append({"role": message.role, "content": blocks})
|
|
143
|
+
return out
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def encode_anthropic_tools(tools: list[Tool]) -> list[dict]:
|
|
147
|
+
out: list[dict] = []
|
|
148
|
+
for t in tools:
|
|
149
|
+
entry: dict = {
|
|
150
|
+
"name": t.name,
|
|
151
|
+
"description": t.description,
|
|
152
|
+
"input_schema": t.input_schema,
|
|
153
|
+
}
|
|
154
|
+
if t.cache:
|
|
155
|
+
# Caches the tools + system prefix up to and including this tool.
|
|
156
|
+
entry["cache_control"] = {"type": "ephemeral"}
|
|
157
|
+
out.append(entry)
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def encode_anthropic_tool_choice(tool_choice: ToolChoice, force_tool: str | None):
|
|
162
|
+
if force_tool is not None:
|
|
163
|
+
return {"type": "tool", "name": force_tool}
|
|
164
|
+
return {
|
|
165
|
+
ToolChoice.AUTO: {"type": "auto"},
|
|
166
|
+
ToolChoice.REQUIRED: {"type": "any"},
|
|
167
|
+
ToolChoice.NONE: {"type": "none"},
|
|
168
|
+
}[tool_choice]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def decode_anthropic_response(raw) -> Response:
|
|
172
|
+
text_chunks: list[str] = []
|
|
173
|
+
tool_calls: list[ToolCall] = []
|
|
174
|
+
for block in raw.content:
|
|
175
|
+
if block.type == "text":
|
|
176
|
+
text_chunks.append(block.text)
|
|
177
|
+
elif block.type == "tool_use":
|
|
178
|
+
tool_calls.append(ToolCall(id=block.id, name=block.name, input=block.input))
|
|
179
|
+
|
|
180
|
+
usage = Usage()
|
|
181
|
+
if getattr(raw, "usage", None) is not None:
|
|
182
|
+
usage.input_tokens = getattr(raw.usage, "input_tokens", None)
|
|
183
|
+
usage.output_tokens = getattr(raw.usage, "output_tokens", None)
|
|
184
|
+
usage.cache_read_tokens = getattr(raw.usage, "cache_read_input_tokens", None)
|
|
185
|
+
usage.cache_write_tokens = getattr(raw.usage, "cache_creation_input_tokens", None)
|
|
186
|
+
|
|
187
|
+
return Response(
|
|
188
|
+
text="".join(text_chunks),
|
|
189
|
+
tool_calls=tool_calls,
|
|
190
|
+
stop_reason=_STOP_REASON.get(raw.stop_reason, StopReason.OTHER),
|
|
191
|
+
usage=usage,
|
|
192
|
+
raw=raw,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class AnthropicAdapter(Adapter):
|
|
197
|
+
def _build_client(self, api_key: str) -> Any:
|
|
198
|
+
anthropic = require_sdk("anthropic", "anthropic")
|
|
199
|
+
return anthropic.Anthropic(api_key=api_key)
|
|
200
|
+
|
|
201
|
+
def _send(self, **create_kwargs: Any) -> Any:
|
|
202
|
+
"""Issue one Messages request over a stream, returning the accumulated
|
|
203
|
+
final message.
|
|
204
|
+
|
|
205
|
+
Streaming rather than `messages.create` because the default cap is the
|
|
206
|
+
model's own maximum (see `CLAUDE_MAX_OUTPUT_TOKENS`), far above the SDK's
|
|
207
|
+
non-streaming ceiling: it derives an expected wall-clock from `max_tokens`
|
|
208
|
+
and raises `ValueError` — before any network call — for a request that
|
|
209
|
+
could outrun its 10-minute timeout. Streaming is the provider's prescribed
|
|
210
|
+
answer for large outputs; it also keeps the connection from idling out on
|
|
211
|
+
a genuinely long generation.
|
|
212
|
+
|
|
213
|
+
`get_final_message()` returns the same `Message` object a non-streaming
|
|
214
|
+
call would, so every caller and `decode_anthropic_response` are unchanged.
|
|
215
|
+
"""
|
|
216
|
+
with self._client.messages.stream(**create_kwargs) as stream:
|
|
217
|
+
return stream.get_final_message()
|
|
218
|
+
|
|
219
|
+
def generate_structured_json(
|
|
220
|
+
self,
|
|
221
|
+
*,
|
|
222
|
+
model: str,
|
|
223
|
+
prompt: str,
|
|
224
|
+
schema: dict,
|
|
225
|
+
system: str | None = None,
|
|
226
|
+
tool_name: str = "generate_json",
|
|
227
|
+
temperature: float = 0.1,
|
|
228
|
+
max_tokens: int | None = None,
|
|
229
|
+
) -> Any:
|
|
230
|
+
max_tokens = resolve_max_tokens(model, max_tokens)
|
|
231
|
+
# NOTE: temperature/top_p/top_k are deliberately NOT forwarded. Newer
|
|
232
|
+
# Anthropic models (Opus 4.7/4.8, Fable 5) reject sampling params with a
|
|
233
|
+
# 400; older ones (Sonnet 4.6, Haiku 4.5) accept them. Support is
|
|
234
|
+
# per-MODEL, not per-provider, so a provider-level capability flag can't
|
|
235
|
+
# express it — see the sampling-params discussion. Omitting them is safe
|
|
236
|
+
# on every model.
|
|
237
|
+
_ = temperature
|
|
238
|
+
create_kwargs: dict[str, Any] = {
|
|
239
|
+
"model": model,
|
|
240
|
+
"max_tokens": max_tokens,
|
|
241
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
242
|
+
"tools": [
|
|
243
|
+
{
|
|
244
|
+
"name": tool_name,
|
|
245
|
+
"description": f"Return the {tool_name} result as structured JSON.",
|
|
246
|
+
"input_schema": schema,
|
|
247
|
+
}
|
|
248
|
+
],
|
|
249
|
+
# Force the tool so the model must emit schema-shaped input.
|
|
250
|
+
"tool_choice": {"type": "tool", "name": tool_name},
|
|
251
|
+
}
|
|
252
|
+
if system:
|
|
253
|
+
create_kwargs["system"] = system
|
|
254
|
+
|
|
255
|
+
response = self._send(**create_kwargs)
|
|
256
|
+
|
|
257
|
+
# A max_tokens stop means the tool-call JSON was cut off mid-object; the
|
|
258
|
+
# partial input would be missing trailing (often required) fields. Fail
|
|
259
|
+
# loudly here rather than letting a downstream validator report a
|
|
260
|
+
# misleading "field required".
|
|
261
|
+
if response.stop_reason == "max_tokens":
|
|
262
|
+
model_max = claude_max_output_tokens(model)
|
|
263
|
+
remedy = (
|
|
264
|
+
" — raise max_tokens toward it."
|
|
265
|
+
if max_tokens < model_max
|
|
266
|
+
else ", so the schema is too large for one response: split the "
|
|
267
|
+
"request or narrow the schema."
|
|
268
|
+
)
|
|
269
|
+
raise StructuredOutputError(
|
|
270
|
+
f"Claude hit the {max_tokens}-token output cap before finishing the "
|
|
271
|
+
f"'{tool_name}' tool call, so the JSON is truncated. This model's "
|
|
272
|
+
f"maximum is {model_max}{remedy}",
|
|
273
|
+
context={
|
|
274
|
+
"provider": self.spec.name,
|
|
275
|
+
"model": model,
|
|
276
|
+
"tool_name": tool_name,
|
|
277
|
+
"max_tokens": max_tokens,
|
|
278
|
+
"model_max_tokens": model_max,
|
|
279
|
+
"stop_reason": response.stop_reason,
|
|
280
|
+
"output_tokens": getattr(response.usage, "output_tokens", None),
|
|
281
|
+
},
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
tool_use = next((b for b in response.content if b.type == "tool_use"), None)
|
|
285
|
+
if tool_use is None:
|
|
286
|
+
raise StructuredOutputError(
|
|
287
|
+
"Claude did not return the forced tool call.",
|
|
288
|
+
context={
|
|
289
|
+
"provider": self.spec.name,
|
|
290
|
+
"model": model,
|
|
291
|
+
"tool_name": tool_name,
|
|
292
|
+
"stop_reason": response.stop_reason,
|
|
293
|
+
},
|
|
294
|
+
)
|
|
295
|
+
# The Anthropic SDK parses tool inputs into native dicts already.
|
|
296
|
+
return tool_use.input
|
|
297
|
+
|
|
298
|
+
def generate_text(
|
|
299
|
+
self,
|
|
300
|
+
*,
|
|
301
|
+
model: str,
|
|
302
|
+
prompt: str,
|
|
303
|
+
system: str | None = None,
|
|
304
|
+
temperature: float = 0.1,
|
|
305
|
+
max_tokens: int | None = None,
|
|
306
|
+
) -> str:
|
|
307
|
+
_ = temperature # see generate_structured_json — not forwarded on Anthropic
|
|
308
|
+
create_kwargs: dict[str, Any] = {
|
|
309
|
+
"model": model,
|
|
310
|
+
"max_tokens": resolve_max_tokens(model, max_tokens),
|
|
311
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
312
|
+
}
|
|
313
|
+
if system:
|
|
314
|
+
create_kwargs["system"] = system
|
|
315
|
+
response = self._send(**create_kwargs)
|
|
316
|
+
return "".join(b.text for b in response.content if b.type == "text")
|
|
317
|
+
|
|
318
|
+
def chat(
|
|
319
|
+
self,
|
|
320
|
+
*,
|
|
321
|
+
model: str,
|
|
322
|
+
messages: list[Message],
|
|
323
|
+
tools: list[Tool] | None = None,
|
|
324
|
+
tool_choice: ToolChoice = ToolChoice.AUTO,
|
|
325
|
+
force_tool: str | None = None,
|
|
326
|
+
system: str | None = None,
|
|
327
|
+
effort: Effort | None = None,
|
|
328
|
+
cache: CacheHandle | None = None,
|
|
329
|
+
temperature: float = 0.1, # not forwarded — see structured-output note
|
|
330
|
+
max_tokens: int | None = None,
|
|
331
|
+
) -> Response:
|
|
332
|
+
_ = temperature
|
|
333
|
+
if cache is not None:
|
|
334
|
+
return self._chat_cached(
|
|
335
|
+
model=model, cache=cache, tail=messages, effort=effort, max_tokens=max_tokens
|
|
336
|
+
)
|
|
337
|
+
system_text, rest = split_system(messages, system)
|
|
338
|
+
create_kwargs: dict[str, Any] = {
|
|
339
|
+
"model": model,
|
|
340
|
+
"max_tokens": resolve_max_tokens(model, max_tokens),
|
|
341
|
+
"messages": encode_anthropic_messages(rest),
|
|
342
|
+
}
|
|
343
|
+
if system_text:
|
|
344
|
+
create_kwargs["system"] = system_text
|
|
345
|
+
if tools:
|
|
346
|
+
create_kwargs["tools"] = encode_anthropic_tools(tools)
|
|
347
|
+
create_kwargs["tool_choice"] = encode_anthropic_tool_choice(
|
|
348
|
+
tool_choice, force_tool
|
|
349
|
+
)
|
|
350
|
+
create_kwargs.update(anthropic_reasoning_kwargs(effort))
|
|
351
|
+
raw = self._send(**create_kwargs)
|
|
352
|
+
return decode_anthropic_response(raw)
|
|
353
|
+
|
|
354
|
+
def create_cache(
|
|
355
|
+
self,
|
|
356
|
+
*,
|
|
357
|
+
model: str,
|
|
358
|
+
messages: list[Message],
|
|
359
|
+
system: str | None = None,
|
|
360
|
+
tools: list[Tool] | None = None,
|
|
361
|
+
ttl: int | None = None,
|
|
362
|
+
) -> CacheHandle:
|
|
363
|
+
"""Anthropic caching is inline and stateless, so there's no server
|
|
364
|
+
resource to create — the handle just captures the prefix to replay with a
|
|
365
|
+
``cache_control`` breakpoint on each use. Pure (no network / no key needed).
|
|
366
|
+
|
|
367
|
+
A prefix below Anthropic's ~1024-token minimum is simply not cached by the
|
|
368
|
+
server (no error), matching the provider's own behavior.
|
|
369
|
+
"""
|
|
370
|
+
return CacheHandle(
|
|
371
|
+
provider=self.spec.name,
|
|
372
|
+
model=model,
|
|
373
|
+
system=system,
|
|
374
|
+
messages=tuple(messages),
|
|
375
|
+
tools=tuple(tools or ()),
|
|
376
|
+
ttl=ttl,
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
def _chat_cached(
|
|
380
|
+
self, *, model: str, cache: CacheHandle, tail: list[Message], effort, max_tokens
|
|
381
|
+
) -> Response:
|
|
382
|
+
"""Replay the cached prefix (system + messages + tools) with a single
|
|
383
|
+
``cache_control`` breakpoint at its deepest segment — which caches the
|
|
384
|
+
whole contiguous prefix above it — then append the dynamic ``tail``."""
|
|
385
|
+
tail_system, tail_rest = split_system(tail, None)
|
|
386
|
+
prefix_messages = list(cache.messages)
|
|
387
|
+
prefix_tools = list(cache.tools)
|
|
388
|
+
# Wire order is tools -> system -> messages; the breakpoint goes on the
|
|
389
|
+
# deepest present segment so everything above it is cached.
|
|
390
|
+
deepest = (
|
|
391
|
+
"messages" if prefix_messages
|
|
392
|
+
else "system" if cache.system
|
|
393
|
+
else "tools" if prefix_tools
|
|
394
|
+
else None
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
encoded_prefix = encode_anthropic_messages(prefix_messages)
|
|
398
|
+
if deepest == "messages" and encoded_prefix and encoded_prefix[-1]["content"]:
|
|
399
|
+
encoded_prefix[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
|
|
400
|
+
create_kwargs: dict[str, Any] = {
|
|
401
|
+
"model": model,
|
|
402
|
+
"max_tokens": resolve_max_tokens(model, max_tokens),
|
|
403
|
+
"messages": encoded_prefix + encode_anthropic_messages(tail_rest),
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
system_blocks: list[dict] = []
|
|
407
|
+
if cache.system:
|
|
408
|
+
block: dict[str, Any] = {"type": "text", "text": cache.system}
|
|
409
|
+
if deepest == "system":
|
|
410
|
+
block["cache_control"] = {"type": "ephemeral"}
|
|
411
|
+
system_blocks.append(block)
|
|
412
|
+
if tail_system: # any system on the tail rides along uncached, after the prefix
|
|
413
|
+
system_blocks.append({"type": "text", "text": tail_system})
|
|
414
|
+
if system_blocks:
|
|
415
|
+
create_kwargs["system"] = system_blocks
|
|
416
|
+
|
|
417
|
+
if prefix_tools:
|
|
418
|
+
encoded_tools = encode_anthropic_tools(prefix_tools)
|
|
419
|
+
if deepest == "tools":
|
|
420
|
+
encoded_tools[-1]["cache_control"] = {"type": "ephemeral"}
|
|
421
|
+
create_kwargs["tools"] = encoded_tools
|
|
422
|
+
|
|
423
|
+
create_kwargs.update(anthropic_reasoning_kwargs(effort))
|
|
424
|
+
raw = self._send(**create_kwargs)
|
|
425
|
+
return decode_anthropic_response(raw)
|
|
426
|
+
|
|
427
|
+
def verify_key(self) -> tuple[bool, str]:
|
|
428
|
+
try:
|
|
429
|
+
self._client.models.list(limit=1)
|
|
430
|
+
return True, "Key is valid."
|
|
431
|
+
except Exception as exc: # noqa: BLE001
|
|
432
|
+
detail = str(exc).strip() or exc.__class__.__name__
|
|
433
|
+
return False, detail.splitlines()[0][:300]
|