inferenceswitch 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,104 @@
1
+ """inferenceswitch — lightweight, import-only multi-provider LLM dispatch.
2
+
3
+ from inferenceswitch import Client
4
+
5
+ client = Client()
6
+ data = client.generate_structured_json(
7
+ model="claude-opus-4-8", # or "gemini-3.5-flash", "groq/llama-3.3-70b", ...
8
+ schema=MyModel.model_json_schema(),
9
+ prompt="...",
10
+ system="...",
11
+ )
12
+
13
+ No proxy, no server, no dynamic provider loading — just a static registry and a
14
+ thick client that handles per-provider structured-output mechanics for you.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ from .adapters.anthropic import (
19
+ CLAUDE_MAX_OUTPUT_TOKENS,
20
+ UNKNOWN_MODEL_MAX_OUTPUT_TOKENS,
21
+ claude_max_output_tokens,
22
+ )
23
+ from .adapters.gemini import (
24
+ GEMINI_MAX_OUTPUT_TOKENS,
25
+ UNKNOWN_GEMINI_MODEL_MAX_OUTPUT_TOKENS,
26
+ gemini_max_output_tokens,
27
+ )
28
+ from .capabilities import Capabilities, Capability, SchemaDialect, StructuredMode
29
+ from .client import Client, ModelChoice, Resolution, env_key_provider
30
+ from .messages import (
31
+ CacheHandle,
32
+ Effort,
33
+ Response,
34
+ Message,
35
+ StopReason,
36
+ Text,
37
+ Tool,
38
+ ToolCall,
39
+ ToolChoice,
40
+ ToolResult,
41
+ ToolUse,
42
+ Usage,
43
+ assistant,
44
+ system,
45
+ tool_result,
46
+ user,
47
+ )
48
+ from .errors import (
49
+ InferenceSwitchError,
50
+ MissingAPIKeyError,
51
+ MissingDependencyError,
52
+ ProviderResolutionError,
53
+ StructuredOutputError,
54
+ UnsupportedCapabilityError,
55
+ )
56
+ from .registry import ProviderSpec, Registry, default_registry
57
+
58
+ __version__ = "0.1.0"
59
+
60
+ __all__ = [
61
+ "Client",
62
+ "ModelChoice",
63
+ "Resolution",
64
+ "env_key_provider",
65
+ "Registry",
66
+ "ProviderSpec",
67
+ "default_registry",
68
+ "Capabilities",
69
+ "Capability",
70
+ "StructuredMode",
71
+ "SchemaDialect",
72
+ "InferenceSwitchError",
73
+ "ProviderResolutionError",
74
+ "MissingAPIKeyError",
75
+ "MissingDependencyError",
76
+ "StructuredOutputError",
77
+ "UnsupportedCapabilityError",
78
+ # chat / tool-calling surface
79
+ "Message",
80
+ "Tool",
81
+ "ToolCall",
82
+ "ToolChoice",
83
+ "ToolUse",
84
+ "ToolResult",
85
+ "Text",
86
+ "CacheHandle",
87
+ "Effort",
88
+ "Response",
89
+ "StopReason",
90
+ "Usage",
91
+ # Per-provider output caps (default = the model's own maximum). Editing a
92
+ # table imposes a house ceiling process-wide; a per-call max_tokens still wins.
93
+ "CLAUDE_MAX_OUTPUT_TOKENS",
94
+ "UNKNOWN_MODEL_MAX_OUTPUT_TOKENS",
95
+ "claude_max_output_tokens",
96
+ "GEMINI_MAX_OUTPUT_TOKENS",
97
+ "UNKNOWN_GEMINI_MODEL_MAX_OUTPUT_TOKENS",
98
+ "gemini_max_output_tokens",
99
+ "user",
100
+ "assistant",
101
+ "system",
102
+ "tool_result",
103
+ "__version__",
104
+ ]
@@ -0,0 +1,34 @@
1
+ """Adapter registry: maps a provider ``kind`` to its adapter class."""
2
+ from __future__ import annotations
3
+
4
+ from ..registry import KIND_ANTHROPIC, KIND_GEMINI, KIND_OPENAI, ProviderSpec
5
+ from .anthropic import AnthropicAdapter
6
+ from .base import Adapter
7
+ from .gemini import GeminiAdapter
8
+ from .openai_compat import OpenAICompatibleAdapter
9
+
10
+ _ADAPTERS_BY_KIND: dict[str, type[Adapter]] = {
11
+ KIND_OPENAI: OpenAICompatibleAdapter,
12
+ KIND_ANTHROPIC: AnthropicAdapter,
13
+ KIND_GEMINI: GeminiAdapter,
14
+ }
15
+
16
+
17
+ def build_adapter(spec: ProviderSpec, api_key: str, *, raw_client=None) -> Adapter:
18
+ try:
19
+ cls = _ADAPTERS_BY_KIND[spec.kind]
20
+ except KeyError:
21
+ raise ValueError(
22
+ f"No adapter registered for provider kind {spec.kind!r} "
23
+ f"(provider {spec.name!r})."
24
+ ) from None
25
+ return cls(spec, api_key, raw_client=raw_client)
26
+
27
+
28
+ __all__ = [
29
+ "Adapter",
30
+ "AnthropicAdapter",
31
+ "GeminiAdapter",
32
+ "OpenAICompatibleAdapter",
33
+ "build_adapter",
34
+ ]
@@ -0,0 +1,433 @@
1
+ """Native Anthropic adapter.
2
+
3
+ Anthropic is not OpenAI-shaped: schema-constrained JSON is obtained by declaring
4
+ the schema as a tool's ``input_schema`` and forcing that tool via ``tool_choice``,
5
+ then reading ``tool_use.input``. Anthropic's schema dialect is the most permissive
6
+ of the three, so no translation is needed on the way in or out.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from typing import Any
11
+
12
+ from ..errors import StructuredOutputError
13
+ from ..messages import (
14
+ CacheHandle,
15
+ Effort,
16
+ Response,
17
+ Message,
18
+ StopReason,
19
+ Text,
20
+ Tool,
21
+ ToolCall,
22
+ ToolChoice,
23
+ ToolResult,
24
+ ToolUse,
25
+ Usage,
26
+ )
27
+ from .base import Adapter, require_sdk, split_system
28
+
29
+ #: Per-model output ceilings for the Claude line. Anthropic requires ``max_tokens``
30
+ #: on every request, so inferenceswitch has to pick a default when the caller doesn't —
31
+ #: and the right default is *that model's* maximum: anything lower silently
32
+ #: truncates a long answer (``stop_reason: "max_tokens"``) for no benefit, since
33
+ #: output is billed per token generated, not per token requested.
34
+ #:
35
+ #: This table is the control surface. It is deliberately a plain mutable dict:
36
+ #: add a row for a model inferenceswitch doesn't know yet, or lower one to put a hard
37
+ #: ceiling on a model process-wide::
38
+ #:
39
+ #: from inferenceswitch import CLAUDE_MAX_OUTPUT_TOKENS
40
+ #: CLAUDE_MAX_OUTPUT_TOKENS["claude-opus-5"] = 32_000 # house limit
41
+ #:
42
+ #: A per-call ``max_tokens=`` still wins over the table, and is never clamped to
43
+ #: it — an explicit number is the caller's decision, and an out-of-range one
44
+ #: surfaces Anthropic's own 400 rather than being silently rewritten.
45
+ CLAUDE_MAX_OUTPUT_TOKENS: dict[str, int] = {
46
+ "claude-fable-5": 128_000,
47
+ "claude-mythos-5": 128_000,
48
+ "claude-opus-5": 128_000,
49
+ "claude-opus-4-8": 128_000,
50
+ "claude-opus-4-7": 128_000,
51
+ "claude-opus-4-6": 128_000,
52
+ "claude-sonnet-5": 128_000,
53
+ "claude-sonnet-4-6": 128_000,
54
+ "claude-haiku-4-5": 64_000,
55
+ }
56
+
57
+ #: Used when a model ID matches nothing in the table. 64000 is the smallest
58
+ #: ceiling in the *current* Claude line (Haiku 4.5), so it is the largest value
59
+ #: that is safe to send blind to any current model. Older models cap lower — add
60
+ #: an explicit row above before using one.
61
+ UNKNOWN_MODEL_MAX_OUTPUT_TOKENS = 64_000
62
+
63
+
64
+ def claude_max_output_tokens(model: str) -> int:
65
+ """This model's maximum output tokens, per :data:`CLAUDE_MAX_OUTPUT_TOKENS`.
66
+
67
+ Falls back to substring matching on the longest known ID so dated snapshots
68
+ (``claude-haiku-4-5-20251001``) and provider-prefixed IDs
69
+ (``anthropic.claude-opus-4-8`` on Bedrock) resolve to the same ceiling as the
70
+ bare alias, then to :data:`UNKNOWN_MODEL_MAX_OUTPUT_TOKENS`.
71
+ """
72
+ exact = CLAUDE_MAX_OUTPUT_TOKENS.get(model)
73
+ if exact is not None:
74
+ return exact
75
+ matches = [name for name in CLAUDE_MAX_OUTPUT_TOKENS if name in model]
76
+ if matches:
77
+ return CLAUDE_MAX_OUTPUT_TOKENS[max(matches, key=len)]
78
+ return UNKNOWN_MODEL_MAX_OUTPUT_TOKENS
79
+
80
+
81
+ def resolve_max_tokens(model: str, max_tokens: int | None) -> int:
82
+ """The caller's ``max_tokens`` if given, else the model's own maximum."""
83
+ return max_tokens if max_tokens is not None else claude_max_output_tokens(model)
84
+
85
+
86
+ _ANTHROPIC_EFFORT = {
87
+ Effort.LOW: "low",
88
+ Effort.MEDIUM: "medium",
89
+ Effort.HIGH: "high",
90
+ Effort.MAX: "max",
91
+ }
92
+
93
+
94
+ def anthropic_reasoning_kwargs(effort: Effort | None) -> dict:
95
+ """Reasoning via adaptive thinking + ``output_config.effort`` on current
96
+ Anthropic models. ``NONE`` disables thinking. (Effort support is per-model —
97
+ e.g. Haiku 4.5 doesn't take it — so a request with effort on a model that
98
+ lacks it will surface the provider's own 400.)"""
99
+ if effort is None:
100
+ return {}
101
+ if effort is Effort.NONE:
102
+ return {"thinking": {"type": "disabled"}}
103
+ return {
104
+ "thinking": {"type": "adaptive"},
105
+ "output_config": {"effort": _ANTHROPIC_EFFORT[effort]},
106
+ }
107
+
108
+ _STOP_REASON = {
109
+ "end_turn": StopReason.END_TURN,
110
+ "max_tokens": StopReason.MAX_TOKENS,
111
+ "tool_use": StopReason.TOOL_USE,
112
+ "stop_sequence": StopReason.STOP_SEQUENCE,
113
+ "refusal": StopReason.REFUSAL,
114
+ }
115
+
116
+
117
+ def encode_anthropic_messages(messages: list[Message]) -> list[dict]:
118
+ """Normalized messages -> Anthropic content-block messages."""
119
+ out: list[dict] = []
120
+ for message in messages:
121
+ blocks: list[dict] = []
122
+ for part in message.parts():
123
+ if isinstance(part, Text):
124
+ if part.text:
125
+ block = {"type": "text", "text": part.text}
126
+ if part.cache:
127
+ block["cache_control"] = {"type": "ephemeral"}
128
+ blocks.append(block)
129
+ elif isinstance(part, ToolUse):
130
+ blocks.append(
131
+ {"type": "tool_use", "id": part.id, "name": part.name, "input": part.input}
132
+ )
133
+ elif isinstance(part, ToolResult):
134
+ blocks.append(
135
+ {
136
+ "type": "tool_result",
137
+ "tool_use_id": part.tool_call_id,
138
+ "content": part.content,
139
+ "is_error": part.is_error,
140
+ }
141
+ )
142
+ out.append({"role": message.role, "content": blocks})
143
+ return out
144
+
145
+
146
+ def encode_anthropic_tools(tools: list[Tool]) -> list[dict]:
147
+ out: list[dict] = []
148
+ for t in tools:
149
+ entry: dict = {
150
+ "name": t.name,
151
+ "description": t.description,
152
+ "input_schema": t.input_schema,
153
+ }
154
+ if t.cache:
155
+ # Caches the tools + system prefix up to and including this tool.
156
+ entry["cache_control"] = {"type": "ephemeral"}
157
+ out.append(entry)
158
+ return out
159
+
160
+
161
+ def encode_anthropic_tool_choice(tool_choice: ToolChoice, force_tool: str | None):
162
+ if force_tool is not None:
163
+ return {"type": "tool", "name": force_tool}
164
+ return {
165
+ ToolChoice.AUTO: {"type": "auto"},
166
+ ToolChoice.REQUIRED: {"type": "any"},
167
+ ToolChoice.NONE: {"type": "none"},
168
+ }[tool_choice]
169
+
170
+
171
+ def decode_anthropic_response(raw) -> Response:
172
+ text_chunks: list[str] = []
173
+ tool_calls: list[ToolCall] = []
174
+ for block in raw.content:
175
+ if block.type == "text":
176
+ text_chunks.append(block.text)
177
+ elif block.type == "tool_use":
178
+ tool_calls.append(ToolCall(id=block.id, name=block.name, input=block.input))
179
+
180
+ usage = Usage()
181
+ if getattr(raw, "usage", None) is not None:
182
+ usage.input_tokens = getattr(raw.usage, "input_tokens", None)
183
+ usage.output_tokens = getattr(raw.usage, "output_tokens", None)
184
+ usage.cache_read_tokens = getattr(raw.usage, "cache_read_input_tokens", None)
185
+ usage.cache_write_tokens = getattr(raw.usage, "cache_creation_input_tokens", None)
186
+
187
+ return Response(
188
+ text="".join(text_chunks),
189
+ tool_calls=tool_calls,
190
+ stop_reason=_STOP_REASON.get(raw.stop_reason, StopReason.OTHER),
191
+ usage=usage,
192
+ raw=raw,
193
+ )
194
+
195
+
196
+ class AnthropicAdapter(Adapter):
197
+ def _build_client(self, api_key: str) -> Any:
198
+ anthropic = require_sdk("anthropic", "anthropic")
199
+ return anthropic.Anthropic(api_key=api_key)
200
+
201
+ def _send(self, **create_kwargs: Any) -> Any:
202
+ """Issue one Messages request over a stream, returning the accumulated
203
+ final message.
204
+
205
+ Streaming rather than `messages.create` because the default cap is the
206
+ model's own maximum (see `CLAUDE_MAX_OUTPUT_TOKENS`), far above the SDK's
207
+ non-streaming ceiling: it derives an expected wall-clock from `max_tokens`
208
+ and raises `ValueError` — before any network call — for a request that
209
+ could outrun its 10-minute timeout. Streaming is the provider's prescribed
210
+ answer for large outputs; it also keeps the connection from idling out on
211
+ a genuinely long generation.
212
+
213
+ `get_final_message()` returns the same `Message` object a non-streaming
214
+ call would, so every caller and `decode_anthropic_response` are unchanged.
215
+ """
216
+ with self._client.messages.stream(**create_kwargs) as stream:
217
+ return stream.get_final_message()
218
+
219
+ def generate_structured_json(
220
+ self,
221
+ *,
222
+ model: str,
223
+ prompt: str,
224
+ schema: dict,
225
+ system: str | None = None,
226
+ tool_name: str = "generate_json",
227
+ temperature: float = 0.1,
228
+ max_tokens: int | None = None,
229
+ ) -> Any:
230
+ max_tokens = resolve_max_tokens(model, max_tokens)
231
+ # NOTE: temperature/top_p/top_k are deliberately NOT forwarded. Newer
232
+ # Anthropic models (Opus 4.7/4.8, Fable 5) reject sampling params with a
233
+ # 400; older ones (Sonnet 4.6, Haiku 4.5) accept them. Support is
234
+ # per-MODEL, not per-provider, so a provider-level capability flag can't
235
+ # express it — see the sampling-params discussion. Omitting them is safe
236
+ # on every model.
237
+ _ = temperature
238
+ create_kwargs: dict[str, Any] = {
239
+ "model": model,
240
+ "max_tokens": max_tokens,
241
+ "messages": [{"role": "user", "content": prompt}],
242
+ "tools": [
243
+ {
244
+ "name": tool_name,
245
+ "description": f"Return the {tool_name} result as structured JSON.",
246
+ "input_schema": schema,
247
+ }
248
+ ],
249
+ # Force the tool so the model must emit schema-shaped input.
250
+ "tool_choice": {"type": "tool", "name": tool_name},
251
+ }
252
+ if system:
253
+ create_kwargs["system"] = system
254
+
255
+ response = self._send(**create_kwargs)
256
+
257
+ # A max_tokens stop means the tool-call JSON was cut off mid-object; the
258
+ # partial input would be missing trailing (often required) fields. Fail
259
+ # loudly here rather than letting a downstream validator report a
260
+ # misleading "field required".
261
+ if response.stop_reason == "max_tokens":
262
+ model_max = claude_max_output_tokens(model)
263
+ remedy = (
264
+ " — raise max_tokens toward it."
265
+ if max_tokens < model_max
266
+ else ", so the schema is too large for one response: split the "
267
+ "request or narrow the schema."
268
+ )
269
+ raise StructuredOutputError(
270
+ f"Claude hit the {max_tokens}-token output cap before finishing the "
271
+ f"'{tool_name}' tool call, so the JSON is truncated. This model's "
272
+ f"maximum is {model_max}{remedy}",
273
+ context={
274
+ "provider": self.spec.name,
275
+ "model": model,
276
+ "tool_name": tool_name,
277
+ "max_tokens": max_tokens,
278
+ "model_max_tokens": model_max,
279
+ "stop_reason": response.stop_reason,
280
+ "output_tokens": getattr(response.usage, "output_tokens", None),
281
+ },
282
+ )
283
+
284
+ tool_use = next((b for b in response.content if b.type == "tool_use"), None)
285
+ if tool_use is None:
286
+ raise StructuredOutputError(
287
+ "Claude did not return the forced tool call.",
288
+ context={
289
+ "provider": self.spec.name,
290
+ "model": model,
291
+ "tool_name": tool_name,
292
+ "stop_reason": response.stop_reason,
293
+ },
294
+ )
295
+ # The Anthropic SDK parses tool inputs into native dicts already.
296
+ return tool_use.input
297
+
298
+ def generate_text(
299
+ self,
300
+ *,
301
+ model: str,
302
+ prompt: str,
303
+ system: str | None = None,
304
+ temperature: float = 0.1,
305
+ max_tokens: int | None = None,
306
+ ) -> str:
307
+ _ = temperature # see generate_structured_json — not forwarded on Anthropic
308
+ create_kwargs: dict[str, Any] = {
309
+ "model": model,
310
+ "max_tokens": resolve_max_tokens(model, max_tokens),
311
+ "messages": [{"role": "user", "content": prompt}],
312
+ }
313
+ if system:
314
+ create_kwargs["system"] = system
315
+ response = self._send(**create_kwargs)
316
+ return "".join(b.text for b in response.content if b.type == "text")
317
+
318
+ def chat(
319
+ self,
320
+ *,
321
+ model: str,
322
+ messages: list[Message],
323
+ tools: list[Tool] | None = None,
324
+ tool_choice: ToolChoice = ToolChoice.AUTO,
325
+ force_tool: str | None = None,
326
+ system: str | None = None,
327
+ effort: Effort | None = None,
328
+ cache: CacheHandle | None = None,
329
+ temperature: float = 0.1, # not forwarded — see structured-output note
330
+ max_tokens: int | None = None,
331
+ ) -> Response:
332
+ _ = temperature
333
+ if cache is not None:
334
+ return self._chat_cached(
335
+ model=model, cache=cache, tail=messages, effort=effort, max_tokens=max_tokens
336
+ )
337
+ system_text, rest = split_system(messages, system)
338
+ create_kwargs: dict[str, Any] = {
339
+ "model": model,
340
+ "max_tokens": resolve_max_tokens(model, max_tokens),
341
+ "messages": encode_anthropic_messages(rest),
342
+ }
343
+ if system_text:
344
+ create_kwargs["system"] = system_text
345
+ if tools:
346
+ create_kwargs["tools"] = encode_anthropic_tools(tools)
347
+ create_kwargs["tool_choice"] = encode_anthropic_tool_choice(
348
+ tool_choice, force_tool
349
+ )
350
+ create_kwargs.update(anthropic_reasoning_kwargs(effort))
351
+ raw = self._send(**create_kwargs)
352
+ return decode_anthropic_response(raw)
353
+
354
+ def create_cache(
355
+ self,
356
+ *,
357
+ model: str,
358
+ messages: list[Message],
359
+ system: str | None = None,
360
+ tools: list[Tool] | None = None,
361
+ ttl: int | None = None,
362
+ ) -> CacheHandle:
363
+ """Anthropic caching is inline and stateless, so there's no server
364
+ resource to create — the handle just captures the prefix to replay with a
365
+ ``cache_control`` breakpoint on each use. Pure (no network / no key needed).
366
+
367
+ A prefix below Anthropic's ~1024-token minimum is simply not cached by the
368
+ server (no error), matching the provider's own behavior.
369
+ """
370
+ return CacheHandle(
371
+ provider=self.spec.name,
372
+ model=model,
373
+ system=system,
374
+ messages=tuple(messages),
375
+ tools=tuple(tools or ()),
376
+ ttl=ttl,
377
+ )
378
+
379
+ def _chat_cached(
380
+ self, *, model: str, cache: CacheHandle, tail: list[Message], effort, max_tokens
381
+ ) -> Response:
382
+ """Replay the cached prefix (system + messages + tools) with a single
383
+ ``cache_control`` breakpoint at its deepest segment — which caches the
384
+ whole contiguous prefix above it — then append the dynamic ``tail``."""
385
+ tail_system, tail_rest = split_system(tail, None)
386
+ prefix_messages = list(cache.messages)
387
+ prefix_tools = list(cache.tools)
388
+ # Wire order is tools -> system -> messages; the breakpoint goes on the
389
+ # deepest present segment so everything above it is cached.
390
+ deepest = (
391
+ "messages" if prefix_messages
392
+ else "system" if cache.system
393
+ else "tools" if prefix_tools
394
+ else None
395
+ )
396
+
397
+ encoded_prefix = encode_anthropic_messages(prefix_messages)
398
+ if deepest == "messages" and encoded_prefix and encoded_prefix[-1]["content"]:
399
+ encoded_prefix[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
400
+ create_kwargs: dict[str, Any] = {
401
+ "model": model,
402
+ "max_tokens": resolve_max_tokens(model, max_tokens),
403
+ "messages": encoded_prefix + encode_anthropic_messages(tail_rest),
404
+ }
405
+
406
+ system_blocks: list[dict] = []
407
+ if cache.system:
408
+ block: dict[str, Any] = {"type": "text", "text": cache.system}
409
+ if deepest == "system":
410
+ block["cache_control"] = {"type": "ephemeral"}
411
+ system_blocks.append(block)
412
+ if tail_system: # any system on the tail rides along uncached, after the prefix
413
+ system_blocks.append({"type": "text", "text": tail_system})
414
+ if system_blocks:
415
+ create_kwargs["system"] = system_blocks
416
+
417
+ if prefix_tools:
418
+ encoded_tools = encode_anthropic_tools(prefix_tools)
419
+ if deepest == "tools":
420
+ encoded_tools[-1]["cache_control"] = {"type": "ephemeral"}
421
+ create_kwargs["tools"] = encoded_tools
422
+
423
+ create_kwargs.update(anthropic_reasoning_kwargs(effort))
424
+ raw = self._send(**create_kwargs)
425
+ return decode_anthropic_response(raw)
426
+
427
+ def verify_key(self) -> tuple[bool, str]:
428
+ try:
429
+ self._client.models.list(limit=1)
430
+ return True, "Key is valid."
431
+ except Exception as exc: # noqa: BLE001
432
+ detail = str(exc).strip() or exc.__class__.__name__
433
+ return False, detail.splitlines()[0][:300]