levier-runtime 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- levier_runtime/__init__.py +57 -0
- levier_runtime/channel/__init__.py +5 -0
- levier_runtime/channel/client.py +317 -0
- levier_runtime/channel/envelope.py +57 -0
- levier_runtime/cli.py +116 -0
- levier_runtime/config.py +289 -0
- levier_runtime/context.py +409 -0
- levier_runtime/events.py +100 -0
- levier_runtime/ids.py +36 -0
- levier_runtime/llm/__init__.py +44 -0
- levier_runtime/llm/anthropic.py +410 -0
- levier_runtime/llm/base.py +134 -0
- levier_runtime/llm/http.py +133 -0
- levier_runtime/llm/openai.py +379 -0
- levier_runtime/llm/router.py +56 -0
- levier_runtime/manager.py +272 -0
- levier_runtime/mcp/__init__.py +8 -0
- levier_runtime/mcp/client.py +289 -0
- levier_runtime/profile.py +91 -0
- levier_runtime/react/__init__.py +11 -0
- levier_runtime/react/agent.py +527 -0
- levier_runtime/react/checkpoint.py +193 -0
- levier_runtime/runtime.py +139 -0
- levier_runtime/toolbox.py +261 -0
- levier_runtime/tools/__init__.py +23 -0
- levier_runtime/tools/base.py +131 -0
- levier_runtime/types.py +143 -0
- levier_runtime-0.1.0.dist-info/METADATA +101 -0
- levier_runtime-0.1.0.dist-info/RECORD +37 -0
- levier_runtime-0.1.0.dist-info/WHEEL +5 -0
- levier_runtime-0.1.0.dist-info/top_level.txt +2 -0
- middleware/__init__.py +1 -0
- middleware/runtime/__init__.py +1 -0
- middleware/runtime/v1/__init__.py +1 -0
- middleware/runtime/v1/runtime_pb2.py +98 -0
- middleware/runtime/v1/runtime_pb2.pyi +332 -0
- middleware/runtime/v1/runtime_pb2_grpc.py +84 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Levier — LLM provider interface.
|
|
2
|
+
|
|
3
|
+
Defines the provider contract consumed by the ReAct runtime, including
|
|
4
|
+
per-run model configuration, streaming deltas, and normalized provider
|
|
5
|
+
errors.
|
|
6
|
+
|
|
7
|
+
Copyright 2026 The Levier Authors.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Protocol, Sequence
|
|
14
|
+
|
|
15
|
+
from middleware.runtime.v1 import runtime_pb2
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class ToolSpec:
|
|
20
|
+
"""One tool definition exposed to the model."""
|
|
21
|
+
|
|
22
|
+
name: str
|
|
23
|
+
description: str = ""
|
|
24
|
+
input_schema_json: str = ""
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def from_manifest(cls, manifest: runtime_pb2.ToolManifest) -> "ToolSpec":
|
|
28
|
+
"""Convert a runtime tool manifest into a provider-facing spec."""
|
|
29
|
+
return cls(
|
|
30
|
+
name=manifest.name,
|
|
31
|
+
description=manifest.description,
|
|
32
|
+
input_schema_json=manifest.input_schema_json,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class ToolCall:
|
|
38
|
+
"""A tool invocation requested by the model."""
|
|
39
|
+
|
|
40
|
+
call_id: str
|
|
41
|
+
name: str
|
|
42
|
+
arguments_json: str = ""
|
|
43
|
+
invocation_id: str = ""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class Message:
|
|
48
|
+
"""One conversation message passed to the provider."""
|
|
49
|
+
|
|
50
|
+
role: str
|
|
51
|
+
content: str = ""
|
|
52
|
+
name: str = ""
|
|
53
|
+
tool_call_id: str = ""
|
|
54
|
+
tool_calls: tuple[ToolCall, ...] = ()
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_proto(cls, message: runtime_pb2.ChatMessage) -> "Message":
|
|
58
|
+
"""Convert a runtime input message."""
|
|
59
|
+
return cls(role=message.role, content=message.content, name=message.name)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class ModelConfig:
|
|
64
|
+
"""Per-run model and credential settings."""
|
|
65
|
+
|
|
66
|
+
provider: str = ""
|
|
67
|
+
model: str = ""
|
|
68
|
+
api_key: str = ""
|
|
69
|
+
base_url: str = ""
|
|
70
|
+
params: dict[str, str] = field(default_factory=dict)
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def from_profile(cls, profile: runtime_pb2.ModelProfile) -> "ModelConfig":
|
|
74
|
+
"""Convert a runtime model profile."""
|
|
75
|
+
return cls(
|
|
76
|
+
provider=profile.provider,
|
|
77
|
+
model=profile.model,
|
|
78
|
+
api_key=profile.api_key,
|
|
79
|
+
base_url=profile.base_url,
|
|
80
|
+
params=dict(profile.params),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class LLMResponse:
|
|
86
|
+
"""One provider completion."""
|
|
87
|
+
|
|
88
|
+
content: str = ""
|
|
89
|
+
tool_calls: list[ToolCall] = field(default_factory=list)
|
|
90
|
+
usage: dict[str, int] = field(default_factory=dict)
|
|
91
|
+
model: str = ""
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def has_output(self) -> bool:
|
|
95
|
+
"""Whether the response contains text or tool calls."""
|
|
96
|
+
return bool(self.content or self.tool_calls)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class LLMError(RuntimeError):
|
|
100
|
+
"""Normalized provider error."""
|
|
101
|
+
|
|
102
|
+
def __init__(
|
|
103
|
+
self,
|
|
104
|
+
code: str,
|
|
105
|
+
message: str,
|
|
106
|
+
*,
|
|
107
|
+
retryable: bool,
|
|
108
|
+
status: int = 0,
|
|
109
|
+
) -> None:
|
|
110
|
+
super().__init__(message)
|
|
111
|
+
self.code = code
|
|
112
|
+
self.retryable = retryable
|
|
113
|
+
self.status = status
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class DeltaCallback(Protocol):
|
|
117
|
+
"""Receives streamed text increments."""
|
|
118
|
+
|
|
119
|
+
async def __call__(self, text: str) -> None:
|
|
120
|
+
"""Handle one text delta."""
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class LLMProvider(Protocol):
|
|
124
|
+
"""Provider contract consumed by the ReAct runtime."""
|
|
125
|
+
|
|
126
|
+
async def complete(
|
|
127
|
+
self,
|
|
128
|
+
messages: Sequence[Message],
|
|
129
|
+
tools: Sequence[ToolSpec],
|
|
130
|
+
model: ModelConfig,
|
|
131
|
+
*,
|
|
132
|
+
on_delta: DeltaCallback | None = None,
|
|
133
|
+
) -> LLMResponse:
|
|
134
|
+
"""Return one completion, optionally streaming text deltas."""
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Levier — minimal streaming HTTP transport for LLM providers.
|
|
2
|
+
|
|
3
|
+
The React Runtime has no third-party HTTP client dependency. This transport uses
|
|
4
|
+
the standard library in a helper thread and exposes an async byte stream so
|
|
5
|
+
the provider can parse server-sent events incrementally.
|
|
6
|
+
|
|
7
|
+
Copyright 2026 The Levier Authors.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import http.client
|
|
14
|
+
import socket
|
|
15
|
+
import threading
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import AsyncIterator
|
|
18
|
+
from urllib.parse import urlsplit
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class HTTPRequest:
|
|
23
|
+
"""One streaming POST request."""
|
|
24
|
+
|
|
25
|
+
url: str
|
|
26
|
+
headers: dict[str, str] = field(default_factory=dict)
|
|
27
|
+
body: bytes = b""
|
|
28
|
+
timeout_s: float = 120.0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class HTTPResponse:
|
|
33
|
+
"""Status, headers, and a streaming response body."""
|
|
34
|
+
|
|
35
|
+
status: int
|
|
36
|
+
headers: dict[str, str]
|
|
37
|
+
body: AsyncIterator[bytes]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class HTTPTransportError(RuntimeError):
|
|
41
|
+
"""Raised when the transport cannot complete a request."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, message: str, *, timeout: bool = False) -> None:
|
|
44
|
+
super().__init__(message)
|
|
45
|
+
self.timeout = timeout
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _is_timeout(exc: object) -> bool:
|
|
49
|
+
return isinstance(exc, (TimeoutError, socket.timeout))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class HTTPTransport:
|
|
53
|
+
"""Transport contract used by providers and tests."""
|
|
54
|
+
|
|
55
|
+
async def open(self, request: HTTPRequest) -> HTTPResponse:
|
|
56
|
+
"""Open a request and return the response stream."""
|
|
57
|
+
raise NotImplementedError
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class UrllibTransport(HTTPTransport):
|
|
61
|
+
"""stdlib HTTP/HTTPS transport backed by a helper thread."""
|
|
62
|
+
|
|
63
|
+
async def open(self, request: HTTPRequest) -> HTTPResponse:
|
|
64
|
+
parts = urlsplit(request.url)
|
|
65
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
66
|
+
raise HTTPTransportError(f"unsupported URL {request.url!r}")
|
|
67
|
+
|
|
68
|
+
loop = asyncio.get_running_loop()
|
|
69
|
+
queue: asyncio.Queue[tuple[str, object]] = asyncio.Queue()
|
|
70
|
+
stop = threading.Event()
|
|
71
|
+
|
|
72
|
+
def put(kind: str, value: object = None) -> None:
|
|
73
|
+
loop.call_soon_threadsafe(queue.put_nowait, (kind, value))
|
|
74
|
+
|
|
75
|
+
def worker() -> None:
|
|
76
|
+
connection = None
|
|
77
|
+
try:
|
|
78
|
+
cls = (
|
|
79
|
+
http.client.HTTPSConnection
|
|
80
|
+
if parts.scheme == "https"
|
|
81
|
+
else http.client.HTTPConnection
|
|
82
|
+
)
|
|
83
|
+
connection = cls(parts.hostname, parts.port, timeout=request.timeout_s)
|
|
84
|
+
path = parts.path or "/"
|
|
85
|
+
if parts.query:
|
|
86
|
+
path = f"{path}?{parts.query}"
|
|
87
|
+
connection.request("POST", path, body=request.body, headers=request.headers)
|
|
88
|
+
response = connection.getresponse()
|
|
89
|
+
headers = {key.lower(): value for key, value in response.getheaders()}
|
|
90
|
+
put("status", (response.status, headers))
|
|
91
|
+
while not stop.is_set():
|
|
92
|
+
chunk = response.read1(65536)
|
|
93
|
+
if not chunk:
|
|
94
|
+
break
|
|
95
|
+
put("chunk", chunk)
|
|
96
|
+
put("done")
|
|
97
|
+
except Exception as exc: # transport failures are normalized by the provider
|
|
98
|
+
put("error", exc)
|
|
99
|
+
finally:
|
|
100
|
+
if connection is not None:
|
|
101
|
+
connection.close()
|
|
102
|
+
|
|
103
|
+
thread = threading.Thread(target=worker, name="llm-http", daemon=True)
|
|
104
|
+
thread.start()
|
|
105
|
+
|
|
106
|
+
kind, value = await queue.get()
|
|
107
|
+
if kind == "error":
|
|
108
|
+
raise HTTPTransportError(str(value), timeout=_is_timeout(value))
|
|
109
|
+
if kind != "status":
|
|
110
|
+
raise HTTPTransportError("transport did not return a response status")
|
|
111
|
+
status, headers = value
|
|
112
|
+
return HTTPResponse(
|
|
113
|
+
status=status,
|
|
114
|
+
headers=headers,
|
|
115
|
+
body=self._body(queue, stop),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
async def _body(
|
|
119
|
+
self,
|
|
120
|
+
queue: asyncio.Queue[tuple[str, object]],
|
|
121
|
+
stop: threading.Event,
|
|
122
|
+
) -> AsyncIterator[bytes]:
|
|
123
|
+
try:
|
|
124
|
+
while True:
|
|
125
|
+
kind, value = await queue.get()
|
|
126
|
+
if kind == "chunk":
|
|
127
|
+
yield value # type: ignore[misc]
|
|
128
|
+
elif kind == "error":
|
|
129
|
+
raise HTTPTransportError(str(value), timeout=_is_timeout(value))
|
|
130
|
+
elif kind == "done":
|
|
131
|
+
break
|
|
132
|
+
finally:
|
|
133
|
+
stop.set()
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""Levier — OpenAI-compatible chat-completions provider (P6A).
|
|
2
|
+
|
|
3
|
+
The first step supports non-streaming JSON responses. SSE/tool-call streaming
|
|
4
|
+
and production credential injection are added in later P6A steps.
|
|
5
|
+
|
|
6
|
+
Copyright 2026 The Levier Authors.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from dataclasses import replace
|
|
14
|
+
from typing import AsyncIterator, Sequence
|
|
15
|
+
|
|
16
|
+
from levier_runtime.llm.base import (
|
|
17
|
+
DeltaCallback,
|
|
18
|
+
LLMError,
|
|
19
|
+
LLMResponse,
|
|
20
|
+
Message,
|
|
21
|
+
ModelConfig,
|
|
22
|
+
ToolCall,
|
|
23
|
+
ToolSpec,
|
|
24
|
+
)
|
|
25
|
+
from levier_runtime.llm.http import (
|
|
26
|
+
HTTPRequest,
|
|
27
|
+
HTTPTransport,
|
|
28
|
+
HTTPTransportError,
|
|
29
|
+
UrllibTransport,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
_DEFAULT_BASE_URL = "https://api.deepseek.com"
|
|
33
|
+
_DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
|
|
34
|
+
_DEFAULT_DEEPSEEK_MODEL = "deepseek-flash"
|
|
35
|
+
_RETRYABLE_STATUS = {408, 409, 429, 500, 502, 503, 504, 529}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class OpenAICompatibleProvider:
|
|
39
|
+
"""Provider for DeepSeek/OpenAI-style /chat/completions APIs."""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
transport: HTTPTransport | None = None,
|
|
44
|
+
*,
|
|
45
|
+
stream: bool = True,
|
|
46
|
+
) -> None:
|
|
47
|
+
self.transport = transport or UrllibTransport()
|
|
48
|
+
self.stream = stream
|
|
49
|
+
|
|
50
|
+
async def complete(
|
|
51
|
+
self,
|
|
52
|
+
messages: Sequence[Message],
|
|
53
|
+
tools: Sequence[ToolSpec],
|
|
54
|
+
model: ModelConfig,
|
|
55
|
+
*,
|
|
56
|
+
on_delta: DeltaCallback | None = None,
|
|
57
|
+
) -> LLMResponse:
|
|
58
|
+
"""Call /chat/completions and parse the JSON response."""
|
|
59
|
+
request = self.build_request(messages, tools, model)
|
|
60
|
+
try:
|
|
61
|
+
response = await self.transport.open(request)
|
|
62
|
+
except HTTPTransportError as exc:
|
|
63
|
+
if getattr(exc, "timeout", False):
|
|
64
|
+
raise LLMError("TIMEOUT", str(exc), retryable=True) from exc
|
|
65
|
+
raise LLMError("PROVIDER_UNAVAILABLE", str(exc), retryable=True) from exc
|
|
66
|
+
|
|
67
|
+
if response.status != 200:
|
|
68
|
+
body = await _read_body(response.body)
|
|
69
|
+
raise _http_error(response.status, body)
|
|
70
|
+
if "application/json" in response.headers.get("content-type", ""):
|
|
71
|
+
body = await _read_body(response.body)
|
|
72
|
+
return _parse_json_response(body, model)
|
|
73
|
+
return await _parse_stream(response.body, model, on_delta)
|
|
74
|
+
|
|
75
|
+
def build_request(
|
|
76
|
+
self,
|
|
77
|
+
messages: Sequence[Message],
|
|
78
|
+
tools: Sequence[ToolSpec],
|
|
79
|
+
model: ModelConfig,
|
|
80
|
+
) -> HTTPRequest:
|
|
81
|
+
"""Build one non-streaming chat-completions request."""
|
|
82
|
+
model = _resolve_model(model)
|
|
83
|
+
if not model.model:
|
|
84
|
+
raise LLMError("BAD_REQUEST", "model is required", retryable=False)
|
|
85
|
+
|
|
86
|
+
body: dict[str, object] = {
|
|
87
|
+
"model": model.model,
|
|
88
|
+
"messages": [_convert_message(message) for message in messages],
|
|
89
|
+
"stream": self.stream,
|
|
90
|
+
}
|
|
91
|
+
if tools:
|
|
92
|
+
body["tools"] = [_tool_schema(tool) for tool in tools]
|
|
93
|
+
body["tool_choice"] = "auto"
|
|
94
|
+
if "temperature" in model.params:
|
|
95
|
+
try:
|
|
96
|
+
body["temperature"] = float(model.params["temperature"])
|
|
97
|
+
except ValueError as exc:
|
|
98
|
+
raise LLMError(
|
|
99
|
+
"BAD_REQUEST",
|
|
100
|
+
"model parameter temperature must be numeric",
|
|
101
|
+
retryable=False,
|
|
102
|
+
) from exc
|
|
103
|
+
if "max_tokens" in model.params:
|
|
104
|
+
try:
|
|
105
|
+
body["max_tokens"] = int(model.params["max_tokens"])
|
|
106
|
+
except ValueError:
|
|
107
|
+
pass
|
|
108
|
+
|
|
109
|
+
headers = {
|
|
110
|
+
"Content-Type": "application/json",
|
|
111
|
+
"Accept": "application/json",
|
|
112
|
+
}
|
|
113
|
+
if model.api_key:
|
|
114
|
+
headers["Authorization"] = f"Bearer {model.api_key}"
|
|
115
|
+
base_url = (model.base_url or _DEFAULT_BASE_URL).rstrip("/")
|
|
116
|
+
return HTTPRequest(
|
|
117
|
+
url=base_url + "/chat/completions",
|
|
118
|
+
headers=headers,
|
|
119
|
+
body=json.dumps(body, ensure_ascii=False).encode("utf-8"),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _resolve_model(model: ModelConfig) -> ModelConfig:
|
|
124
|
+
"""Apply local provider environment fallbacks when fields are empty."""
|
|
125
|
+
provider = (model.provider or "").lower()
|
|
126
|
+
if provider not in {"deepseek", "openai"}:
|
|
127
|
+
return model
|
|
128
|
+
if provider == "deepseek":
|
|
129
|
+
env_key = "DEEPSEEK_API_KEY"
|
|
130
|
+
env_base = "DEEPSEEK_BASE_URL"
|
|
131
|
+
env_model = "DEEPSEEK_MODEL"
|
|
132
|
+
default_base = _DEFAULT_BASE_URL
|
|
133
|
+
default_model = _DEFAULT_DEEPSEEK_MODEL
|
|
134
|
+
else:
|
|
135
|
+
env_key = "OPENAI_API_KEY"
|
|
136
|
+
env_base = "OPENAI_BASE_URL"
|
|
137
|
+
env_model = "OPENAI_MODEL"
|
|
138
|
+
default_base = _DEFAULT_OPENAI_BASE_URL
|
|
139
|
+
default_model = ""
|
|
140
|
+
api_key = model.api_key or os.environ.get(env_key, "").strip()
|
|
141
|
+
if not api_key:
|
|
142
|
+
raise LLMError(
|
|
143
|
+
"PROVIDER_UNAVAILABLE",
|
|
144
|
+
f"{env_key} is required for provider {provider}",
|
|
145
|
+
retryable=False,
|
|
146
|
+
)
|
|
147
|
+
base_url = model.base_url or os.environ.get(env_base, "").strip() or default_base
|
|
148
|
+
model_name = model.model or os.environ.get(env_model, "").strip() or default_model
|
|
149
|
+
if not model_name:
|
|
150
|
+
raise LLMError("BAD_REQUEST", "model is required", retryable=False)
|
|
151
|
+
return replace(model, api_key=api_key, base_url=base_url, model=model_name)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _convert_message(message: Message) -> dict[str, object]:
|
|
155
|
+
converted: dict[str, object] = {
|
|
156
|
+
"role": message.role,
|
|
157
|
+
"content": message.content,
|
|
158
|
+
}
|
|
159
|
+
if message.role == "tool" and message.tool_call_id:
|
|
160
|
+
converted["tool_call_id"] = message.tool_call_id
|
|
161
|
+
if message.role == "assistant" and message.tool_calls:
|
|
162
|
+
converted["tool_calls"] = [
|
|
163
|
+
{
|
|
164
|
+
"id": call.call_id,
|
|
165
|
+
"type": "function",
|
|
166
|
+
"function": {
|
|
167
|
+
"name": call.name,
|
|
168
|
+
"arguments": call.arguments_json or "{}",
|
|
169
|
+
},
|
|
170
|
+
}
|
|
171
|
+
for call in message.tool_calls
|
|
172
|
+
]
|
|
173
|
+
return converted
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _tool_schema(tool: ToolSpec) -> dict[str, object]:
|
|
177
|
+
try:
|
|
178
|
+
schema = json.loads(tool.input_schema_json or "{}")
|
|
179
|
+
except json.JSONDecodeError as exc:
|
|
180
|
+
raise LLMError(
|
|
181
|
+
"BAD_REQUEST",
|
|
182
|
+
f"tool {tool.name!r} has an invalid input schema",
|
|
183
|
+
retryable=False,
|
|
184
|
+
) from exc
|
|
185
|
+
return {
|
|
186
|
+
"type": "function",
|
|
187
|
+
"function": {
|
|
188
|
+
"name": tool.name,
|
|
189
|
+
"description": tool.description,
|
|
190
|
+
"parameters": schema or {"type": "object"},
|
|
191
|
+
},
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _parse_json_response(body: bytes, model: ModelConfig) -> LLMResponse:
|
|
196
|
+
try:
|
|
197
|
+
payload = json.loads(body.decode("utf-8"))
|
|
198
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
199
|
+
raise LLMError("BAD_RESPONSE", "provider returned invalid JSON", retryable=False) from exc
|
|
200
|
+
|
|
201
|
+
choices = payload.get("choices")
|
|
202
|
+
if not isinstance(choices, list) or not choices:
|
|
203
|
+
raise LLMError("BAD_RESPONSE", "provider returned no choices", retryable=False)
|
|
204
|
+
message = choices[0].get("message") or {}
|
|
205
|
+
content = message.get("content") or ""
|
|
206
|
+
tool_calls: list[ToolCall] = []
|
|
207
|
+
for call in message.get("tool_calls") or []:
|
|
208
|
+
function = call.get("function") or {}
|
|
209
|
+
arguments = function.get("arguments", "{}")
|
|
210
|
+
if not isinstance(arguments, str):
|
|
211
|
+
arguments = json.dumps(arguments, ensure_ascii=False, separators=(",", ":"))
|
|
212
|
+
tool_calls.append(
|
|
213
|
+
ToolCall(
|
|
214
|
+
call_id=str(call.get("id") or ""),
|
|
215
|
+
name=str(function.get("name") or ""),
|
|
216
|
+
arguments_json=arguments,
|
|
217
|
+
)
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
usage: dict[str, int] = {}
|
|
221
|
+
raw_usage = payload.get("usage")
|
|
222
|
+
if isinstance(raw_usage, dict):
|
|
223
|
+
for key, value in raw_usage.items():
|
|
224
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
225
|
+
usage[key] = value
|
|
226
|
+
return LLMResponse(
|
|
227
|
+
content=str(content),
|
|
228
|
+
tool_calls=tool_calls,
|
|
229
|
+
usage=usage,
|
|
230
|
+
model=str(payload.get("model") or model.model),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
async def _parse_stream(
|
|
235
|
+
chunks: AsyncIterator[bytes],
|
|
236
|
+
model: ModelConfig,
|
|
237
|
+
on_delta: DeltaCallback | None,
|
|
238
|
+
) -> LLMResponse:
|
|
239
|
+
text_parts: list[str] = []
|
|
240
|
+
tool_calls: dict[int, dict[str, str]] = {}
|
|
241
|
+
usage: dict[str, int] = {}
|
|
242
|
+
response_model = model.model
|
|
243
|
+
|
|
244
|
+
async for event in _iter_sse(chunks):
|
|
245
|
+
if event.get("error"):
|
|
246
|
+
raise _stream_error(event)
|
|
247
|
+
if event.get("model"):
|
|
248
|
+
response_model = str(event["model"])
|
|
249
|
+
raw_usage = event.get("usage")
|
|
250
|
+
if isinstance(raw_usage, dict):
|
|
251
|
+
for key, value in raw_usage.items():
|
|
252
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
253
|
+
usage[key] = value
|
|
254
|
+
for choice in event.get("choices") or []:
|
|
255
|
+
delta = choice.get("delta") or {}
|
|
256
|
+
content = delta.get("content")
|
|
257
|
+
if content:
|
|
258
|
+
text_parts.append(str(content))
|
|
259
|
+
if on_delta is not None:
|
|
260
|
+
await on_delta(str(content))
|
|
261
|
+
for call in delta.get("tool_calls") or []:
|
|
262
|
+
index = int(call.get("index", 0))
|
|
263
|
+
entry = tool_calls.setdefault(index, {
|
|
264
|
+
"call_id": "",
|
|
265
|
+
"name": "",
|
|
266
|
+
"arguments": "",
|
|
267
|
+
})
|
|
268
|
+
if call.get("id"):
|
|
269
|
+
entry["call_id"] = str(call["id"])
|
|
270
|
+
function = call.get("function") or {}
|
|
271
|
+
if function.get("name"):
|
|
272
|
+
entry["name"] = str(function["name"])
|
|
273
|
+
arguments = function.get("arguments")
|
|
274
|
+
if arguments:
|
|
275
|
+
entry["arguments"] += str(arguments)
|
|
276
|
+
|
|
277
|
+
return LLMResponse(
|
|
278
|
+
content="".join(text_parts),
|
|
279
|
+
tool_calls=[
|
|
280
|
+
ToolCall(
|
|
281
|
+
call_id=entry["call_id"] or f"call_{index}",
|
|
282
|
+
name=entry["name"],
|
|
283
|
+
arguments_json=entry["arguments"] or "{}",
|
|
284
|
+
)
|
|
285
|
+
for index, entry in sorted(tool_calls.items())
|
|
286
|
+
],
|
|
287
|
+
usage=usage,
|
|
288
|
+
model=response_model,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
async def _iter_sse(chunks: AsyncIterator[bytes]) -> AsyncIterator[dict[str, object]]:
|
|
293
|
+
buffer = ""
|
|
294
|
+
async for chunk in chunks:
|
|
295
|
+
buffer += chunk.decode("utf-8", errors="replace")
|
|
296
|
+
while "\n" in buffer:
|
|
297
|
+
line, buffer = buffer.split("\n", 1)
|
|
298
|
+
line = line.rstrip("\r")
|
|
299
|
+
if not line or line.startswith(":"):
|
|
300
|
+
continue
|
|
301
|
+
if not line.startswith("data:"):
|
|
302
|
+
continue
|
|
303
|
+
data = line[len("data:"):].strip()
|
|
304
|
+
if data == "[DONE]":
|
|
305
|
+
return
|
|
306
|
+
try:
|
|
307
|
+
parsed = json.loads(data)
|
|
308
|
+
except json.JSONDecodeError as exc:
|
|
309
|
+
raise LLMError(
|
|
310
|
+
"BAD_RESPONSE",
|
|
311
|
+
"provider returned invalid SSE JSON",
|
|
312
|
+
retryable=False,
|
|
313
|
+
) from exc
|
|
314
|
+
if isinstance(parsed, dict):
|
|
315
|
+
yield parsed
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
async def _read_body(chunks: AsyncIterator[bytes]) -> bytes:
|
|
319
|
+
parts: list[bytes] = []
|
|
320
|
+
try:
|
|
321
|
+
async for chunk in chunks:
|
|
322
|
+
parts.append(chunk)
|
|
323
|
+
finally:
|
|
324
|
+
aclose = getattr(chunks, "aclose", None)
|
|
325
|
+
if aclose is not None:
|
|
326
|
+
await aclose()
|
|
327
|
+
return b"".join(parts)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _http_error(status: int, body: bytes) -> LLMError:
|
|
331
|
+
code = "PROVIDER_UNAVAILABLE" if status in _RETRYABLE_STATUS else "PROVIDER_ERROR"
|
|
332
|
+
if status in {401, 403}:
|
|
333
|
+
code = "AUTH_FAILED"
|
|
334
|
+
elif status == 408:
|
|
335
|
+
code = "TIMEOUT"
|
|
336
|
+
elif status == 429:
|
|
337
|
+
code = "RATE_LIMITED"
|
|
338
|
+
elif status in {400, 404, 422}:
|
|
339
|
+
code = "BAD_REQUEST"
|
|
340
|
+
message = f"provider returned HTTP {status}"
|
|
341
|
+
try:
|
|
342
|
+
payload = json.loads(body.decode("utf-8"))
|
|
343
|
+
error = payload.get("error") or {}
|
|
344
|
+
if isinstance(error, dict) and error.get("message"):
|
|
345
|
+
message = str(error["message"])
|
|
346
|
+
mapped = _ERROR_CODE_BY_TYPE.get(str(error.get("type", ""))) if isinstance(error, dict) else None
|
|
347
|
+
if mapped:
|
|
348
|
+
code = mapped
|
|
349
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
350
|
+
pass
|
|
351
|
+
retryable = code in {"TIMEOUT", "RATE_LIMITED", "PROVIDER_UNAVAILABLE"}
|
|
352
|
+
return LLMError(code, message, retryable=retryable, status=status)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _stream_error(event: dict[str, object]) -> LLMError:
|
|
356
|
+
error = event.get("error") or {}
|
|
357
|
+
if not isinstance(error, dict):
|
|
358
|
+
error = {}
|
|
359
|
+
error_type = str(error.get("type", ""))
|
|
360
|
+
code = _ERROR_CODE_BY_TYPE.get(error_type, "PROVIDER_ERROR")
|
|
361
|
+
return LLMError(
|
|
362
|
+
code,
|
|
363
|
+
str(error.get("message") or "provider stream error"),
|
|
364
|
+
retryable=code in {"TIMEOUT", "RATE_LIMITED", "PROVIDER_UNAVAILABLE"},
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
_ERROR_CODE_BY_TYPE = {
|
|
369
|
+
"authentication_error": "AUTH_FAILED",
|
|
370
|
+
"invalid_api_key": "AUTH_FAILED",
|
|
371
|
+
"invalid_request_error": "BAD_REQUEST",
|
|
372
|
+
"rate_limit_error": "RATE_LIMITED",
|
|
373
|
+
"rate_limit_exceeded": "RATE_LIMITED",
|
|
374
|
+
"overloaded_error": "PROVIDER_UNAVAILABLE",
|
|
375
|
+
"server_error": "PROVIDER_UNAVAILABLE",
|
|
376
|
+
"timeout": "TIMEOUT",
|
|
377
|
+
"timeout_error": "TIMEOUT",
|
|
378
|
+
}
|
|
379
|
+
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Levier — LLM provider routing.
|
|
2
|
+
|
|
3
|
+
Dispatches one completion by the provider named in ModelConfig.
|
|
4
|
+
|
|
5
|
+
Copyright 2026 The Levier Authors.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Sequence
|
|
11
|
+
|
|
12
|
+
from levier_runtime.llm.base import (
|
|
13
|
+
DeltaCallback,
|
|
14
|
+
LLMError,
|
|
15
|
+
LLMProvider,
|
|
16
|
+
LLMResponse,
|
|
17
|
+
Message,
|
|
18
|
+
ModelConfig,
|
|
19
|
+
ToolSpec,
|
|
20
|
+
)
|
|
21
|
+
from levier_runtime.llm.anthropic import AnthropicProvider
|
|
22
|
+
from levier_runtime.llm.openai import OpenAICompatibleProvider
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ProviderRouter:
|
|
26
|
+
"""Route completions to Anthropic or an OpenAI-compatible provider."""
|
|
27
|
+
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
anthropic: LLMProvider | None = None,
|
|
31
|
+
openai: LLMProvider | None = None,
|
|
32
|
+
) -> None:
|
|
33
|
+
self.anthropic = anthropic or AnthropicProvider()
|
|
34
|
+
self.openai = openai or OpenAICompatibleProvider()
|
|
35
|
+
|
|
36
|
+
async def complete(
|
|
37
|
+
self,
|
|
38
|
+
messages: Sequence[Message],
|
|
39
|
+
tools: Sequence[ToolSpec],
|
|
40
|
+
model: ModelConfig,
|
|
41
|
+
*,
|
|
42
|
+
on_delta: DeltaCallback | None = None,
|
|
43
|
+
) -> LLMResponse:
|
|
44
|
+
"""Dispatch by model.provider."""
|
|
45
|
+
provider = (model.provider or "anthropic").lower()
|
|
46
|
+
if provider == "anthropic":
|
|
47
|
+
target = self.anthropic
|
|
48
|
+
elif provider in {"deepseek", "openai"}:
|
|
49
|
+
target = self.openai
|
|
50
|
+
else:
|
|
51
|
+
raise LLMError(
|
|
52
|
+
"BAD_REQUEST",
|
|
53
|
+
f"unsupported model provider {model.provider!r}",
|
|
54
|
+
retryable=False,
|
|
55
|
+
)
|
|
56
|
+
return await target.complete(messages, tools, model, on_delta=on_delta)
|