langchain-llama-cpp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_llama_cpp/__init__.py +27 -0
- langchain_llama_cpp/_wire.py +328 -0
- langchain_llama_cpp/chat_models.py +1350 -0
- langchain_llama_cpp/content_blocks.py +73 -0
- langchain_llama_cpp/exceptions.py +197 -0
- langchain_llama_cpp/py.typed +0 -0
- langchain_llama_cpp/utils.py +34 -0
- langchain_llama_cpp-0.1.0.dist-info/METADATA +248 -0
- langchain_llama_cpp-0.1.0.dist-info/RECORD +11 -0
- langchain_llama_cpp-0.1.0.dist-info/WHEEL +4 -0
- langchain_llama_cpp-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Content-block translator for llama-server messages (`model_provider` "llama-cpp").
|
|
2
|
+
|
|
3
|
+
Core builds `content_blocks` for providers without a translator by best-effort
|
|
4
|
+
parsing, and that is what we want, except when streaming with
|
|
5
|
+
`output_version="v1"`. Core then numbers blocks that have no `index` with a
|
|
6
|
+
counter that goes up whenever the block type changes (reasoning 0, text 1, ...),
|
|
7
|
+
while a tool-call chunk keeps the server's tool-call index (0, 1, ...). Equal
|
|
8
|
+
indexes are merged as one block, so the first tool call was folded into the
|
|
9
|
+
reasoning or text block before it and vanished from `content_blocks`.
|
|
10
|
+
|
|
11
|
+
For streamed chunks the translator returns core's blocks with two changes:
|
|
12
|
+
|
|
13
|
+
- tool-call chunks get `lc_tc_<i>` indexes, which can't equal a counter value
|
|
14
|
+
(`merge_lists` merges `lc_` indexes only with the same string). Reasoning and
|
|
15
|
+
text keep core's counter, so text after a tool call stays a separate block.
|
|
16
|
+
- every chunk that carries tool-call fragments gets them as `tool_call_chunk`
|
|
17
|
+
blocks. Core does that only when the chunk has no text; otherwise it adds a
|
|
18
|
+
partially parsed `tool_call` that would survive next to the finished call.
|
|
19
|
+
|
|
20
|
+
The blocks are copies: reading `content_blocks` never changes the message.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from langchain_core.messages import AIMessage, AIMessageChunk
|
|
26
|
+
from langchain_core.messages import content as types
|
|
27
|
+
from langchain_core.messages.block_translators import register_translator
|
|
28
|
+
|
|
29
|
+
MODEL_PROVIDER = "llama-cpp"
|
|
30
|
+
|
|
31
|
+
_TOOL_CALL_TYPES = {"tool_call", "invalid_tool_call", "tool_call_chunk"}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def translate_content(message: AIMessage) -> list[types.ContentBlock]:
|
|
35
|
+
"""Core's best-effort blocks, unchanged (a whole message isn't merged).
|
|
36
|
+
|
|
37
|
+
A copy of the message without our `model_provider` skips the registry, so
|
|
38
|
+
core falls back to its own parsing instead of calling us again.
|
|
39
|
+
"""
|
|
40
|
+
metadata = {
|
|
41
|
+
key: value
|
|
42
|
+
for key, value in message.response_metadata.items()
|
|
43
|
+
if key != "model_provider"
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return message.model_copy(update={"response_metadata": metadata}).content_blocks
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]:
|
|
50
|
+
"""Blocks for a streamed chunk: core's, with tool-call chunks re-indexed."""
|
|
51
|
+
blocks: list[dict[str, Any]] = [dict(block) for block in translate_content(message)]
|
|
52
|
+
|
|
53
|
+
if message.tool_call_chunks and message.chunk_position != "last":
|
|
54
|
+
blocks = [block for block in blocks if block["type"] not in _TOOL_CALL_TYPES]
|
|
55
|
+
|
|
56
|
+
for position, chunk in enumerate(message.tool_call_chunks):
|
|
57
|
+
# Without the server's index, the chunk's position stands in, so the
|
|
58
|
+
# block never falls back to core's colliding counter.
|
|
59
|
+
index = chunk.get("index")
|
|
60
|
+
blocks.append(
|
|
61
|
+
{
|
|
62
|
+
"type": "tool_call_chunk",
|
|
63
|
+
"id": chunk.get("id"),
|
|
64
|
+
"name": chunk.get("name"),
|
|
65
|
+
"args": chunk.get("args"),
|
|
66
|
+
"index": f"lc_tc_{position if index is None else index}",
|
|
67
|
+
}
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
return blocks # pyright: ignore[reportReturnType]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
register_translator(MODEL_PROVIDER, translate_content, translate_content_chunk)
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Errors raised by ChatLlama.
|
|
2
|
+
|
|
3
|
+
Each class inherits from both the httpx exception it replaces and the matching
|
|
4
|
+
`langchain_core.exceptions.ModelError` subclass, so `except httpx.HTTPStatusError`
|
|
5
|
+
and `except ContextOverflowError` both work. `is_retryable` comes from core.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Any, cast
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
from langchain_core.exceptions import (
|
|
12
|
+
ContextOverflowError,
|
|
13
|
+
ModelAPIError,
|
|
14
|
+
ModelAuthenticationError,
|
|
15
|
+
ModelConnectionError,
|
|
16
|
+
ModelInvalidRequestError,
|
|
17
|
+
ModelNotFoundError,
|
|
18
|
+
ModelPermissionDeniedError,
|
|
19
|
+
ModelRateLimitError,
|
|
20
|
+
ModelTimeoutError,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
from langchain_llama_cpp._wire import LlamaError
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LlamaContextOverflowError(httpx.HTTPStatusError, ContextOverflowError):
|
|
27
|
+
"""The prompt is longer than the server's context window (HTTP 400).
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
n_prompt_tokens: Tokens in the rejected prompt, as counted by the server.
|
|
31
|
+
n_ctx: The server's context window.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
message: str,
|
|
37
|
+
*,
|
|
38
|
+
request: httpx.Request,
|
|
39
|
+
response: httpx.Response,
|
|
40
|
+
n_prompt_tokens: int | None = None,
|
|
41
|
+
n_ctx: int | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""Create the error with the server's token counts."""
|
|
44
|
+
super().__init__(message, request=request, response=response)
|
|
45
|
+
self.n_prompt_tokens = n_prompt_tokens
|
|
46
|
+
self.n_ctx = n_ctx
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class LlamaInvalidRequestError(httpx.HTTPStatusError, ModelInvalidRequestError):
|
|
50
|
+
"""The server rejected the request (HTTP 400 and other 4xx)."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class LlamaAuthenticationError(httpx.HTTPStatusError, ModelAuthenticationError):
|
|
54
|
+
"""The server needs an API key, or rejected the one sent (HTTP 401)."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class LlamaPermissionDeniedError(httpx.HTTPStatusError, ModelPermissionDeniedError):
|
|
58
|
+
"""The server refused the request (HTTP 403)."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class LlamaModelNotFoundError(httpx.HTTPStatusError, ModelNotFoundError):
|
|
62
|
+
"""llama-swap has no entry with this `model` name (HTTP 404)."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class LlamaRateLimitError(httpx.HTTPStatusError, ModelRateLimitError):
|
|
66
|
+
"""The server has no free slot (HTTP 429)."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class LlamaAPIError(httpx.HTTPStatusError, ModelAPIError):
|
|
70
|
+
"""The server failed (HTTP 5xx)."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class LlamaConnectionError(httpx.TransportError, ModelConnectionError):
|
|
74
|
+
"""The server could not be reached, or the connection broke."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class LlamaTimeoutError(httpx.TimeoutException, ModelTimeoutError):
|
|
78
|
+
"""The request took longer than `timeout`."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
_STATUS_ERRORS: dict[int, type[httpx.HTTPStatusError]] = {
|
|
82
|
+
401: LlamaAuthenticationError,
|
|
83
|
+
403: LlamaPermissionDeniedError,
|
|
84
|
+
404: LlamaModelNotFoundError,
|
|
85
|
+
429: LlamaRateLimitError,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def error_from_response(response: httpx.Response) -> httpx.HTTPStatusError:
|
|
90
|
+
"""Build the right error for an HTTP error response."""
|
|
91
|
+
# llama-server: {"error": {"type", "message", ...}}; llama-swap: plain text.
|
|
92
|
+
try:
|
|
93
|
+
body: object = response.json()
|
|
94
|
+
except ValueError:
|
|
95
|
+
body = None
|
|
96
|
+
|
|
97
|
+
raw_detail = _json_object(body).get("error")
|
|
98
|
+
detail = cast("LlamaError", _json_object(raw_detail))
|
|
99
|
+
|
|
100
|
+
reason = detail.get("message") or raw_detail or response.text
|
|
101
|
+
message = f"HTTP {response.status_code} from {response.request.url}: {reason}"
|
|
102
|
+
|
|
103
|
+
return _build_error(
|
|
104
|
+
message,
|
|
105
|
+
detail,
|
|
106
|
+
response.status_code,
|
|
107
|
+
response,
|
|
108
|
+
fallback=LlamaInvalidRequestError, # e.g. a 3xx that wasn't followed
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def error_from_stream_event(
|
|
113
|
+
detail: object, response: httpx.Response
|
|
114
|
+
) -> httpx.HTTPStatusError:
|
|
115
|
+
"""Build the error for a `data: {"error": ...}` event in a running stream.
|
|
116
|
+
|
|
117
|
+
The HTTP status was already 200, so the class comes from the event's
|
|
118
|
+
`type` and `code` (llama-server's `format_error_response`).
|
|
119
|
+
"""
|
|
120
|
+
fields = cast("LlamaError", _json_object(detail))
|
|
121
|
+
reason = fields.get("message") or detail or "error during streaming"
|
|
122
|
+
message = f"Error during streaming from {response.request.url}: {reason}"
|
|
123
|
+
|
|
124
|
+
code = fields.get("code")
|
|
125
|
+
status = code if isinstance(code, int) else None
|
|
126
|
+
|
|
127
|
+
return _build_error(message, fields, status, response, fallback=LlamaAPIError)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def error_from_cut_off_stream(response: httpx.Response) -> LlamaConnectionError:
|
|
131
|
+
"""The error for a stream that ended without `data: [DONE]` or an error event.
|
|
132
|
+
|
|
133
|
+
llama-server always ends a finished stream with `[DONE]`, so the
|
|
134
|
+
connection was closed early (a crash, a restart, or a proxy in between).
|
|
135
|
+
"""
|
|
136
|
+
return LlamaConnectionError(
|
|
137
|
+
message=f"Stream from {response.request.url} ended without data: [DONE]",
|
|
138
|
+
request=response.request,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def error_from_transport(exc: httpx.TransportError) -> httpx.TransportError:
|
|
143
|
+
"""Wrap an httpx transport failure in the matching ChatLlama error."""
|
|
144
|
+
try:
|
|
145
|
+
request: httpx.Request | None = exc.request
|
|
146
|
+
except RuntimeError: # httpx raises this when no request is attached
|
|
147
|
+
request = None
|
|
148
|
+
|
|
149
|
+
cls = (
|
|
150
|
+
LlamaTimeoutError
|
|
151
|
+
if isinstance(exc, httpx.TimeoutException)
|
|
152
|
+
else LlamaConnectionError
|
|
153
|
+
)
|
|
154
|
+
return cls(str(exc) or type(exc).__name__, request=request)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _build_error(
|
|
158
|
+
message: str,
|
|
159
|
+
detail: LlamaError,
|
|
160
|
+
status: int | None,
|
|
161
|
+
response: httpx.Response,
|
|
162
|
+
*,
|
|
163
|
+
fallback: type[httpx.HTTPStatusError],
|
|
164
|
+
) -> httpx.HTTPStatusError:
|
|
165
|
+
"""Pick the error class from llama-server's error `type` and the status.
|
|
166
|
+
|
|
167
|
+
Overflow is recognised by `type`, then the status decides: a mapped code,
|
|
168
|
+
any other 4xx (invalid request), 5xx (API error), else `fallback`.
|
|
169
|
+
"""
|
|
170
|
+
request = response.request
|
|
171
|
+
|
|
172
|
+
if detail.get("type") == "exceed_context_size_error":
|
|
173
|
+
return LlamaContextOverflowError(
|
|
174
|
+
message,
|
|
175
|
+
request=request,
|
|
176
|
+
response=response,
|
|
177
|
+
n_prompt_tokens=detail.get("n_prompt_tokens"),
|
|
178
|
+
n_ctx=detail.get("n_ctx"),
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
if status is None:
|
|
182
|
+
cls = fallback
|
|
183
|
+
elif status in _STATUS_ERRORS:
|
|
184
|
+
cls = _STATUS_ERRORS[status]
|
|
185
|
+
elif 400 <= status < 500: # noqa: PLR2004
|
|
186
|
+
cls = LlamaInvalidRequestError
|
|
187
|
+
elif status >= 500: # noqa: PLR2004
|
|
188
|
+
cls = LlamaAPIError
|
|
189
|
+
else:
|
|
190
|
+
cls = fallback
|
|
191
|
+
|
|
192
|
+
return cls(message, request=request, response=response)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _json_object(value: object) -> dict[str, Any]:
|
|
196
|
+
"""`value` if it's a JSON object, else an empty dict."""
|
|
197
|
+
return cast("dict[str, Any]", value) if isinstance(value, dict) else {}
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Helpers for working with messages from reasoning models."""
|
|
2
|
+
|
|
3
|
+
from textwrap import indent
|
|
4
|
+
|
|
5
|
+
from langchain_core.messages import AIMessage, BaseMessage
|
|
6
|
+
from langchain_core.utils.interactive_env import is_interactive_env
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def pretty_repr(message: BaseMessage, *, html: bool = False) -> str:
|
|
10
|
+
"""Like `message.pretty_repr()`, but shows an AI message's reasoning.
|
|
11
|
+
|
|
12
|
+
Reasoning is read from `content_blocks`, so it works for reasoning stored in
|
|
13
|
+
`additional_kwargs["reasoning_content"]` (ChatLlama, ChatDeepSeek, ChatOllama,
|
|
14
|
+
ChatGroq) and for reasoning blocks in `content` (`output_version="v1"`,
|
|
15
|
+
ChatAnthropic). Non-AI messages are passed to their own `pretty_repr`.
|
|
16
|
+
"""
|
|
17
|
+
if not isinstance(message, AIMessage):
|
|
18
|
+
return message.pretty_repr(html=html)
|
|
19
|
+
parts = [
|
|
20
|
+
block.get("reasoning", "")
|
|
21
|
+
for block in message.content_blocks
|
|
22
|
+
if block["type"] == "reasoning"
|
|
23
|
+
]
|
|
24
|
+
reasoning = "\n\n".join(part for part in parts if part)
|
|
25
|
+
# Core prints list content as a raw repr, so hand it text only, with the
|
|
26
|
+
# reasoning section in front; core still adds the title, name and tool calls.
|
|
27
|
+
section = f"Reasoning:\n{indent(reasoning, ' ')}" if reasoning else ""
|
|
28
|
+
content = "\n\n".join(part for part in (section, message.text) if part)
|
|
29
|
+
return message.model_copy(update={"content": content}).pretty_repr(html=html)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def pretty_print(message: BaseMessage) -> None:
|
|
33
|
+
"""Print `pretty_repr(message)`, with a bold title in Jupyter."""
|
|
34
|
+
print(pretty_repr(message, html=is_interactive_env())) # noqa: T201
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: langchain-llama-cpp
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An integration package connecting llama.cpp's llama-server and LangChain, with reasoning, timings, structured output and tool calling
|
|
5
|
+
Keywords: langchain,llama.cpp,llama-server,llama-swap,llm,local-llm,chat-model
|
|
6
|
+
Author: Bharathan Mudaliar
|
|
7
|
+
Author-email: Bharathan Mudaliar <bharathanmudaliar@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Dist: httpx>=0.27,<1
|
|
21
|
+
Requires-Dist: langchain-core>=1.6,<2
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Project-URL: Repository, https://github.com/bharat029/langchain-llama-cpp
|
|
24
|
+
Project-URL: Issues, https://github.com/bharat029/langchain-llama-cpp/issues
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# langchain-llama-cpp
|
|
28
|
+
|
|
29
|
+
[](https://pypi.org/project/langchain-llama-cpp/)
|
|
30
|
+
[](https://opensource.org/licenses/MIT)
|
|
31
|
+
|
|
32
|
+
A LangChain chat model, `ChatLlama`, for llama.cpp's `llama-server`. It's built
|
|
33
|
+
on `langchain-core` and `httpx` only and talks to `/v1/chat/completions`
|
|
34
|
+
directly, so nothing llama-server sends back is lost on the way.
|
|
35
|
+
|
|
36
|
+
**Why not `ChatOpenAI`?** llama-server speaks the OpenAI format, but with extras
|
|
37
|
+
that `ChatOpenAI` drops or gets wrong:
|
|
38
|
+
|
|
39
|
+
- it drops `reasoning_content` (the model's thinking) and llama-server's `timings`;
|
|
40
|
+
- it sends a named `tool_choice` as an object, which llama-server silently turns
|
|
41
|
+
into `"auto"`;
|
|
42
|
+
- it can't send llama.cpp's own sampling fields (`top_k`, `min_p`, `samplers`,
|
|
43
|
+
DRY, XTC, ...) as typed parameters.
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install langchain-llama-cpp
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
or `uv add langchain-llama-cpp`. Python 3.10+.
|
|
52
|
+
|
|
53
|
+
## Tested setup
|
|
54
|
+
|
|
55
|
+
- **llama-swap** in front of llama-server. `model` is the llama-swap entry to
|
|
56
|
+
use. A bare llama-server (no llama-swap) is not tested yet.
|
|
57
|
+
- llama-server build **b11429**, with gemma 4, Qwen3.6 and gpt-oss, on
|
|
58
|
+
langchain-core 1.6.
|
|
59
|
+
|
|
60
|
+
## Quick start
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from langchain_llama_cpp import ChatLlama
|
|
64
|
+
|
|
65
|
+
llm = ChatLlama(model="gemma", max_tokens=2000) # base_url from LLAMA_BASE_URL
|
|
66
|
+
msg = llm.invoke("What is 17 * 23? Answer with the number only.")
|
|
67
|
+
|
|
68
|
+
msg.content # '391'
|
|
69
|
+
msg.additional_kwargs["reasoning_content"] # 'The user wants to know the product...'
|
|
70
|
+
msg.usage_metadata # {'input_tokens': 32, 'output_tokens': 223, ...}
|
|
71
|
+
msg.response_metadata["timings"] # {'prompt_n': 25, 'predicted_n': 223, ...}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Set `max_tokens` with room for thinking. Reasoning length varies from run to
|
|
75
|
+
run: gemma answered this in 223 tokens once, and in another run used all of
|
|
76
|
+
`max_tokens=300` and ended mid-thought, with an empty `content` and
|
|
77
|
+
`finish_reason: "length"`.
|
|
78
|
+
|
|
79
|
+
`base_url` defaults to the `LLAMA_BASE_URL` environment variable, e.g.
|
|
80
|
+
`http://localhost:8080`. Pass `base_url=` to override it.
|
|
81
|
+
|
|
82
|
+
Reasoning is stored where ChatDeepSeek and ChatOllama put it,
|
|
83
|
+
`additional_kwargs["reasoning_content"]`, so `msg.content_blocks` shows it as a
|
|
84
|
+
standard `{"type": "reasoning"}` block. `langchain_llama_cpp.utils.pretty_print(msg)`
|
|
85
|
+
prints a message with its reasoning.
|
|
86
|
+
|
|
87
|
+
## Usage
|
|
88
|
+
|
|
89
|
+
### Thinking
|
|
90
|
+
|
|
91
|
+
The switch lives in the chat template, and its name differs per model family:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
ChatLlama(model="gemma", chat_template_kwargs={"enable_thinking": False}) # gemma, qwen
|
|
95
|
+
ChatLlama(model="gpt-oss", chat_template_kwargs={"reasoning_effort": "low"}) # gpt-oss
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Thinking tokens count towards `max_tokens` (see Quick start).
|
|
99
|
+
|
|
100
|
+
When you send a conversation back, an `AIMessage`'s reasoning goes back too, and
|
|
101
|
+
the model's chat template decides whether to show it to the model.
|
|
102
|
+
|
|
103
|
+
### Streaming and async
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
for chunk in llm.stream("Count to five."):
|
|
107
|
+
print(chunk.content, end="")
|
|
108
|
+
|
|
109
|
+
msg = await llm.ainvoke("Hello!")
|
|
110
|
+
async for chunk in llm.astream("Hello!"):
|
|
111
|
+
...
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Reasoning arrives in `chunk.additional_kwargs["reasoning_content"]`. Usage and
|
|
115
|
+
timings come in the last chunk (`stream_usage=True`, the default).
|
|
116
|
+
|
|
117
|
+
### Tool calling
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from langchain_core.tools import tool
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@tool
|
|
124
|
+
def get_weather(city: str) -> str:
|
|
125
|
+
"""Get the current weather in a city."""
|
|
126
|
+
return f"Sunny in {city}"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
llm_with_tools = llm.bind_tools([get_weather])
|
|
130
|
+
msg = llm_with_tools.invoke("What's the weather in Paris?")
|
|
131
|
+
msg.tool_calls # [{'name': 'get_weather', 'args': {'city': 'Paris'}, 'id': '...', ...}]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
llama-server accepts only the strings `"auto"`, `"none"` and `"required"` as
|
|
135
|
+
`tool_choice`. A tool's name (or `"any"`, or `True`) is sent as `"required"`:
|
|
136
|
+
the model must call a tool but may pick a different one, because the server
|
|
137
|
+
can't force a particular tool.
|
|
138
|
+
|
|
139
|
+
### Structured output
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from pydantic import BaseModel, Field
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class Person(BaseModel):
|
|
146
|
+
name: str
|
|
147
|
+
age: int = Field(description="Age in years")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
structured = llm.with_structured_output(Person)
|
|
151
|
+
structured.invoke("Alice is 30.") # Person(name='Alice', age=30)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Three methods:
|
|
155
|
+
|
|
156
|
+
| `method` | What's sent | The model sees the schema? |
|
|
157
|
+
|---|---|---|
|
|
158
|
+
| `"function_calling"` (default) | the schema as the only tool, `tool_choice: "required"` | yes, through the chat template |
|
|
159
|
+
| `"json_schema"` | `response_format: {"type": "json_schema", ...}` | no: it's enforced by the grammar only |
|
|
160
|
+
| `"json_mode"` | `response_format: {"type": "json_object"}` | no |
|
|
161
|
+
|
|
162
|
+
`function_calling` is the default because with `json_schema` the model never
|
|
163
|
+
sees the schema's field descriptions: asked for an age described as "in months",
|
|
164
|
+
it fills in years. Pass `include_format_instructions=True` with `json_schema` or
|
|
165
|
+
`json_mode` to add the schema to the system message.
|
|
166
|
+
|
|
167
|
+
### Request parameters
|
|
168
|
+
|
|
169
|
+
Every field `/v1/chat/completions` accepts in b11429 is a typed parameter:
|
|
170
|
+
`temperature`, `top_k`, `top_p`, `min_p`, `samplers`, `repeat_penalty`, the DRY
|
|
171
|
+
and XTC samplers, `grammar`, `json_schema`, `reasoning_budget_tokens`,
|
|
172
|
+
`cache_prompt`, `id_slot`, `lora` and more. Set them on the model or per call:
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
llm = ChatLlama(model="gemma", temperature=0.2, top_k=40)
|
|
176
|
+
llm.invoke("Hi", max_tokens=500)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
They all default to `None`, which leaves them out of the request so the
|
|
180
|
+
server's per-model defaults apply. To see those defaults:
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
llm.get_server_defaults()
|
|
184
|
+
# {'temperature': 1.0, 'top_k': 64, 'top_p': 0.95, 'max_tokens': -1, ..., 'n_ctx': 131072}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
### Errors and retries
|
|
188
|
+
|
|
189
|
+
Errors are subclasses of both the httpx exception and LangChain's `ModelError`
|
|
190
|
+
classes, so either `except` works:
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from langchain_llama_cpp import LlamaContextOverflowError
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
llm.invoke(very_long_prompt)
|
|
197
|
+
except LlamaContextOverflowError as error:
|
|
198
|
+
print(error.n_prompt_tokens, error.n_ctx)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
There are also `LlamaModelNotFoundError` (an unknown llama-swap entry),
|
|
202
|
+
`LlamaInvalidRequestError`, `LlamaRateLimitError`, `LlamaAPIError` (5xx),
|
|
203
|
+
`LlamaConnectionError` and `LlamaTimeoutError`.
|
|
204
|
+
|
|
205
|
+
Connection errors, timeouts, 429 and 5xx are retried `max_retries` times
|
|
206
|
+
(default 2) with exponential backoff, obeying `Retry-After`. `timeout` is in
|
|
207
|
+
seconds and defaults to `None` (no timeout), since local generation can be slow.
|
|
208
|
+
|
|
209
|
+
## Known limitations
|
|
210
|
+
|
|
211
|
+
- **Forcing a tool with Qwen3.5/3.6.** With `tool_choice="required"` (also used
|
|
212
|
+
by `with_structured_output`'s default method), llama-server's grammar for these
|
|
213
|
+
models lets the model write text before the call and won't let it stop
|
|
214
|
+
without one. A model that doesn't want a tool keeps writing until
|
|
215
|
+
`max_tokens`, and there's no tool call. This is
|
|
216
|
+
[ggml-org/llama.cpp#27217](https://github.com/ggml-org/llama.cpp/issues/27217);
|
|
217
|
+
keep `max_tokens` set.
|
|
218
|
+
- **A named `tool_choice`** is sent as `"required"` (see Tool calling).
|
|
219
|
+
- **JSON schema bounds on numbers** (`minimum`, `maximum`, `multipleOf` on
|
|
220
|
+
`number`) aren't enforced by llama-server's grammar. Integer bounds are.
|
|
221
|
+
- **Bare llama-server** (without llama-swap) isn't tested; `get_server_defaults`
|
|
222
|
+
needs llama-swap's `/upstream/<model>/props`.
|
|
223
|
+
|
|
224
|
+
## Development
|
|
225
|
+
|
|
226
|
+
Bug reports and pull requests are welcome on
|
|
227
|
+
[GitHub](https://github.com/bharat029/langchain-llama-cpp/issues).
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
uv sync
|
|
231
|
+
uv run pytest tests/unit_tests # offline
|
|
232
|
+
uv run ruff check && uv run ruff format --check && uv run pyright
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
The integration tests (LangChain's standard `langchain-tests` suite, on gemma,
|
|
236
|
+
qwen and gpt-oss) need a llama-swap server. Copy `.env.example` to `.env` and
|
|
237
|
+
set `LLAMA_BASE_URL`, then:
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
uv run --env-file .env pytest tests/integration_tests
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
The JSON llama-server sends back is described in
|
|
244
|
+
`src/langchain_llama_cpp/_wire.py`.
|
|
245
|
+
|
|
246
|
+
## License
|
|
247
|
+
|
|
248
|
+
MIT
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
langchain_llama_cpp/__init__.py,sha256=XsmABAlyu2VPq63HO8kgsQnMejrAABizhzTahnza5Kg,699
|
|
2
|
+
langchain_llama_cpp/_wire.py,sha256=e_RLwgm3B4r4fKjZD4sqqpBKqakOnJrNjnoY9Aid27o,9662
|
|
3
|
+
langchain_llama_cpp/chat_models.py,sha256=9hHqPOU0VeTxEn0OdbdLunsglFXupaI3g98jnWhd_cc,51749
|
|
4
|
+
langchain_llama_cpp/content_blocks.py,sha256=Wrq9fay9XAR49049cM2xW6wbSuqMTBI5ZPPBg00BXB0,3223
|
|
5
|
+
langchain_llama_cpp/exceptions.py,sha256=IBLY9ksRGz8ln7q13CgIExpv8EI0ZJnN5dpOe8egrYo,6413
|
|
6
|
+
langchain_llama_cpp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
langchain_llama_cpp/utils.py,sha256=51-_rpeHFTWHysM6gOGvMUJ0-cmc-7kwrGh2vE3Sryc,1563
|
|
8
|
+
langchain_llama_cpp-0.1.0.dist-info/licenses/LICENSE,sha256=RfhLrWQ1wQUzta_PGhka9c4cR6sNf8KuUZtc83a8Usc,1075
|
|
9
|
+
langchain_llama_cpp-0.1.0.dist-info/WHEEL,sha256=4OL6Foqnnp3xRY5wMkjgc25_i5YJC6dKsC6LPcjqEoU,80
|
|
10
|
+
langchain_llama_cpp-0.1.0.dist-info/METADATA,sha256=6Y1eDoWzMsZSkQSWOBZYB5ZslKARnJpN0WIZF2pF3v8,8963
|
|
11
|
+
langchain_llama_cpp-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bharathan Mudaliar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|