langchain-llama-cpp 0.1.0__tar.gz

langchain_llama_cpp-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bharathan Mudaliar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_llama_cpp-0.1.0/PKG-INFO

@@ -0,0 +1,248 @@
+Metadata-Version: 2.4
+Name: langchain-llama-cpp
+Version: 0.1.0
+Summary: An integration package connecting llama.cpp's llama-server and LangChain, with reasoning, timings, structured output and tool calling
+Keywords: langchain,llama.cpp,llama-server,llama-swap,llm,local-llm,chat-model
+Author: Bharathan Mudaliar
+Author-email: Bharathan Mudaliar <bharathanmudaliar@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Dist: httpx>=0.27,<1
+Requires-Dist: langchain-core>=1.6,<2
+Requires-Python: >=3.10
+Project-URL: Repository, https://github.com/bharat029/langchain-llama-cpp
+Project-URL: Issues, https://github.com/bharat029/langchain-llama-cpp/issues
+Description-Content-Type: text/markdown
+
+# langchain-llama-cpp
+
+[![PyPI - Version](https://img.shields.io/pypi/v/langchain-llama-cpp?label=%20)](https://pypi.org/project/langchain-llama-cpp/)
+[![PyPI - License](https://img.shields.io/pypi/l/langchain-llama-cpp)](https://opensource.org/licenses/MIT)
+
+A LangChain chat model, `ChatLlama`, for llama.cpp's `llama-server`. It's built
+on `langchain-core` and `httpx` only and talks to `/v1/chat/completions`
+directly, so nothing llama-server sends back is lost on the way.
+
+**Why not `ChatOpenAI`?** llama-server speaks the OpenAI format, but with extras
+that `ChatOpenAI` drops or gets wrong:
+
+- it drops `reasoning_content` (the model's thinking) and llama-server's `timings`;
+- it sends a named `tool_choice` as an object, which llama-server silently turns
+  into `"auto"`;
+- it can't send llama.cpp's own sampling fields (`top_k`, `min_p`, `samplers`,
+  DRY, XTC, ...) as typed parameters.
+
+## Install
+
+```bash
+pip install langchain-llama-cpp
+```
+
+or `uv add langchain-llama-cpp`. Python 3.10+.
+
+## Tested setup
+
+- **llama-swap** in front of llama-server. `model` is the llama-swap entry to
+  use. A bare llama-server (no llama-swap) is not tested yet.
+- llama-server build **b11429**, with gemma 4, Qwen3.6 and gpt-oss, on
+  langchain-core 1.6.
+
+## Quick start
+
+```python
+from langchain_llama_cpp import ChatLlama
+
+llm = ChatLlama(model="gemma", max_tokens=2000)  # base_url from LLAMA_BASE_URL
+msg = llm.invoke("What is 17 * 23? Answer with the number only.")
+
+msg.content  # '391'
+msg.additional_kwargs["reasoning_content"]  # 'The user wants to know the product...'
+msg.usage_metadata  # {'input_tokens': 32, 'output_tokens': 223, ...}
+msg.response_metadata["timings"]  # {'prompt_n': 25, 'predicted_n': 223, ...}
+```
+
+Set `max_tokens` with room for thinking. Reasoning length varies from run to
+run: gemma answered this in 223 tokens once, and in another run used all of
+`max_tokens=300` and ended mid-thought, with an empty `content` and
+`finish_reason: "length"`.
+
+`base_url` defaults to the `LLAMA_BASE_URL` environment variable, e.g.
+`http://localhost:8080`. Pass `base_url=` to override it.
+
+Reasoning is stored where ChatDeepSeek and ChatOllama put it,
+`additional_kwargs["reasoning_content"]`, so `msg.content_blocks` shows it as a
+standard `{"type": "reasoning"}` block. `langchain_llama_cpp.utils.pretty_print(msg)`
+prints a message with its reasoning.
+
+## Usage
+
+### Thinking
+
+The switch lives in the chat template, and its name differs per model family:
+
+```python
+ChatLlama(model="gemma", chat_template_kwargs={"enable_thinking": False})  # gemma, qwen
+ChatLlama(model="gpt-oss", chat_template_kwargs={"reasoning_effort": "low"})  # gpt-oss
+```
+
+Thinking tokens count towards `max_tokens` (see Quick start).
+
+When you send a conversation back, an `AIMessage`'s reasoning goes back too, and
+the model's chat template decides whether to show it to the model.
+
+### Streaming and async
+
+```python
+for chunk in llm.stream("Count to five."):
+    print(chunk.content, end="")
+
+msg = await llm.ainvoke("Hello!")
+async for chunk in llm.astream("Hello!"):
+    ...
+```
+
+Reasoning arrives in `chunk.additional_kwargs["reasoning_content"]`. Usage and
+timings come in the last chunk (`stream_usage=True`, the default).
+
+### Tool calling
+
+```python
+from langchain_core.tools import tool
+
+
+@tool
+def get_weather(city: str) -> str:
+    """Get the current weather in a city."""
+    return f"Sunny in {city}"
+
+
+llm_with_tools = llm.bind_tools([get_weather])
+msg = llm_with_tools.invoke("What's the weather in Paris?")
+msg.tool_calls  # [{'name': 'get_weather', 'args': {'city': 'Paris'}, 'id': '...', ...}]
+```
+
+llama-server accepts only the strings `"auto"`, `"none"` and `"required"` as
+`tool_choice`. A tool's name (or `"any"`, or `True`) is sent as `"required"`:
+the model must call a tool but may pick a different one, because the server
+can't force a particular tool.
+
+### Structured output
+
+```python
+from pydantic import BaseModel, Field
+
+
+class Person(BaseModel):
+    name: str
+    age: int = Field(description="Age in years")
+
+
+structured = llm.with_structured_output(Person)
+structured.invoke("Alice is 30.")  # Person(name='Alice', age=30)
+```
+
+Three methods:
+
+| `method` | What's sent | The model sees the schema? |
+|---|---|---|
+| `"function_calling"` (default) | the schema as the only tool, `tool_choice: "required"` | yes, through the chat template |
+| `"json_schema"` | `response_format: {"type": "json_schema", ...}` | no: it's enforced by the grammar only |
+| `"json_mode"` | `response_format: {"type": "json_object"}` | no |
+
+`function_calling` is the default because with `json_schema` the model never
+sees the schema's field descriptions: asked for an age described as "in months",
+it fills in years. Pass `include_format_instructions=True` with `json_schema` or
+`json_mode` to add the schema to the system message.
+
+### Request parameters
+
+Every field `/v1/chat/completions` accepts in b11429 is a typed parameter:
+`temperature`, `top_k`, `top_p`, `min_p`, `samplers`, `repeat_penalty`, the DRY
+and XTC samplers, `grammar`, `json_schema`, `reasoning_budget_tokens`,
+`cache_prompt`, `id_slot`, `lora` and more. Set them on the model or per call:
+
+```python
+llm = ChatLlama(model="gemma", temperature=0.2, top_k=40)
+llm.invoke("Hi", max_tokens=500)
+```
+
+They all default to `None`, which leaves them out of the request so the
+server's per-model defaults apply. To see those defaults:
+
+```python
+llm.get_server_defaults()
+# {'temperature': 1.0, 'top_k': 64, 'top_p': 0.95, 'max_tokens': -1, ..., 'n_ctx': 131072}
+```
+
+### Errors and retries
+
+Errors are subclasses of both the httpx exception and LangChain's `ModelError`
+classes, so either `except` works:
+
+```python
+from langchain_llama_cpp import LlamaContextOverflowError
+
+try:
+    llm.invoke(very_long_prompt)
+except LlamaContextOverflowError as error:
+    print(error.n_prompt_tokens, error.n_ctx)
+```
+
+There are also `LlamaModelNotFoundError` (an unknown llama-swap entry),
+`LlamaInvalidRequestError`, `LlamaRateLimitError`, `LlamaAPIError` (5xx),
+`LlamaConnectionError` and `LlamaTimeoutError`.
+
+Connection errors, timeouts, 429 and 5xx are retried `max_retries` times
+(default 2) with exponential backoff, obeying `Retry-After`. `timeout` is in
+seconds and defaults to `None` (no timeout), since local generation can be slow.
+
+## Known limitations
+
+- **Forcing a tool with Qwen3.5/3.6.** With `tool_choice="required"` (also used
+  by `with_structured_output`'s default method), llama-server's grammar for these
+  models lets the model write text before the call and won't let it stop
+  without one. A model that doesn't want a tool keeps writing until
+  `max_tokens`, and there's no tool call. This is
+  [ggml-org/llama.cpp#27217](https://github.com/ggml-org/llama.cpp/issues/27217);
+  keep `max_tokens` set.
+- **A named `tool_choice`** is sent as `"required"` (see Tool calling).
+- **JSON schema bounds on numbers** (`minimum`, `maximum`, `multipleOf` on
+  `number`) aren't enforced by llama-server's grammar. Integer bounds are.
+- **Bare llama-server** (without llama-swap) isn't tested; `get_server_defaults`
+  needs llama-swap's `/upstream/<model>/props`.
+
+## Development
+
+Bug reports and pull requests are welcome on
+[GitHub](https://github.com/bharat029/langchain-llama-cpp/issues).
+
+```bash
+uv sync
+uv run pytest tests/unit_tests          # offline
+uv run ruff check && uv run ruff format --check && uv run pyright
+```
+
+The integration tests (LangChain's standard `langchain-tests` suite, on gemma,
+qwen and gpt-oss) need a llama-swap server. Copy `.env.example` to `.env` and
+set `LLAMA_BASE_URL`, then:
+
+```bash
+uv run --env-file .env pytest tests/integration_tests
+```
+
+The JSON llama-server sends back is described in
+`src/langchain_llama_cpp/_wire.py`.
+
+## License
+
+MIT

langchain_llama_cpp-0.1.0/README.md

@@ -0,0 +1,222 @@
+# langchain-llama-cpp
+
+[![PyPI - Version](https://img.shields.io/pypi/v/langchain-llama-cpp?label=%20)](https://pypi.org/project/langchain-llama-cpp/)
+[![PyPI - License](https://img.shields.io/pypi/l/langchain-llama-cpp)](https://opensource.org/licenses/MIT)
+
+A LangChain chat model, `ChatLlama`, for llama.cpp's `llama-server`. It's built
+on `langchain-core` and `httpx` only and talks to `/v1/chat/completions`
+directly, so nothing llama-server sends back is lost on the way.
+
+**Why not `ChatOpenAI`?** llama-server speaks the OpenAI format, but with extras
+that `ChatOpenAI` drops or gets wrong:
+
+- it drops `reasoning_content` (the model's thinking) and llama-server's `timings`;
+- it sends a named `tool_choice` as an object, which llama-server silently turns
+  into `"auto"`;
+- it can't send llama.cpp's own sampling fields (`top_k`, `min_p`, `samplers`,
+  DRY, XTC, ...) as typed parameters.
+
+## Install
+
+```bash
+pip install langchain-llama-cpp
+```
+
+or `uv add langchain-llama-cpp`. Python 3.10+.
+
+## Tested setup
+
+- **llama-swap** in front of llama-server. `model` is the llama-swap entry to
+  use. A bare llama-server (no llama-swap) is not tested yet.
+- llama-server build **b11429**, with gemma 4, Qwen3.6 and gpt-oss, on
+  langchain-core 1.6.
+
+## Quick start
+
+```python
+from langchain_llama_cpp import ChatLlama
+
+llm = ChatLlama(model="gemma", max_tokens=2000)  # base_url from LLAMA_BASE_URL
+msg = llm.invoke("What is 17 * 23? Answer with the number only.")
+
+msg.content  # '391'
+msg.additional_kwargs["reasoning_content"]  # 'The user wants to know the product...'
+msg.usage_metadata  # {'input_tokens': 32, 'output_tokens': 223, ...}
+msg.response_metadata["timings"]  # {'prompt_n': 25, 'predicted_n': 223, ...}
+```
+
+Set `max_tokens` with room for thinking. Reasoning length varies from run to
+run: gemma answered this in 223 tokens once, and in another run used all of
+`max_tokens=300` and ended mid-thought, with an empty `content` and
+`finish_reason: "length"`.
+
+`base_url` defaults to the `LLAMA_BASE_URL` environment variable, e.g.
+`http://localhost:8080`. Pass `base_url=` to override it.
+
+Reasoning is stored where ChatDeepSeek and ChatOllama put it,
+`additional_kwargs["reasoning_content"]`, so `msg.content_blocks` shows it as a
+standard `{"type": "reasoning"}` block. `langchain_llama_cpp.utils.pretty_print(msg)`
+prints a message with its reasoning.
+
+## Usage
+
+### Thinking
+
+The switch lives in the chat template, and its name differs per model family:
+
+```python
+ChatLlama(model="gemma", chat_template_kwargs={"enable_thinking": False})  # gemma, qwen
+ChatLlama(model="gpt-oss", chat_template_kwargs={"reasoning_effort": "low"})  # gpt-oss
+```
+
+Thinking tokens count towards `max_tokens` (see Quick start).
+
+When you send a conversation back, an `AIMessage`'s reasoning goes back too, and
+the model's chat template decides whether to show it to the model.
+
+### Streaming and async
+
+```python
+for chunk in llm.stream("Count to five."):
+    print(chunk.content, end="")
+
+msg = await llm.ainvoke("Hello!")
+async for chunk in llm.astream("Hello!"):
+    ...
+```
+
+Reasoning arrives in `chunk.additional_kwargs["reasoning_content"]`. Usage and
+timings come in the last chunk (`stream_usage=True`, the default).
+
+### Tool calling
+
+```python
+from langchain_core.tools import tool
+
+
+@tool
+def get_weather(city: str) -> str:
+    """Get the current weather in a city."""
+    return f"Sunny in {city}"
+
+
+llm_with_tools = llm.bind_tools([get_weather])
+msg = llm_with_tools.invoke("What's the weather in Paris?")
+msg.tool_calls  # [{'name': 'get_weather', 'args': {'city': 'Paris'}, 'id': '...', ...}]
+```
+
+llama-server accepts only the strings `"auto"`, `"none"` and `"required"` as
+`tool_choice`. A tool's name (or `"any"`, or `True`) is sent as `"required"`:
+the model must call a tool but may pick a different one, because the server
+can't force a particular tool.
+
+### Structured output
+
+```python
+from pydantic import BaseModel, Field
+
+
+class Person(BaseModel):
+    name: str
+    age: int = Field(description="Age in years")
+
+
+structured = llm.with_structured_output(Person)
+structured.invoke("Alice is 30.")  # Person(name='Alice', age=30)
+```
+
+Three methods:
+
+| `method` | What's sent | The model sees the schema? |
+|---|---|---|
+| `"function_calling"` (default) | the schema as the only tool, `tool_choice: "required"` | yes, through the chat template |
+| `"json_schema"` | `response_format: {"type": "json_schema", ...}` | no: it's enforced by the grammar only |
+| `"json_mode"` | `response_format: {"type": "json_object"}` | no |
+
+`function_calling` is the default because with `json_schema` the model never
+sees the schema's field descriptions: asked for an age described as "in months",
+it fills in years. Pass `include_format_instructions=True` with `json_schema` or
+`json_mode` to add the schema to the system message.
+
+### Request parameters
+
+Every field `/v1/chat/completions` accepts in b11429 is a typed parameter:
+`temperature`, `top_k`, `top_p`, `min_p`, `samplers`, `repeat_penalty`, the DRY
+and XTC samplers, `grammar`, `json_schema`, `reasoning_budget_tokens`,
+`cache_prompt`, `id_slot`, `lora` and more. Set them on the model or per call:
+
+```python
+llm = ChatLlama(model="gemma", temperature=0.2, top_k=40)
+llm.invoke("Hi", max_tokens=500)
+```
+
+They all default to `None`, which leaves them out of the request so the
+server's per-model defaults apply. To see those defaults:
+
+```python
+llm.get_server_defaults()
+# {'temperature': 1.0, 'top_k': 64, 'top_p': 0.95, 'max_tokens': -1, ..., 'n_ctx': 131072}
+```
+
+### Errors and retries
+
+Errors are subclasses of both the httpx exception and LangChain's `ModelError`
+classes, so either `except` works:
+
+```python
+from langchain_llama_cpp import LlamaContextOverflowError
+
+try:
+    llm.invoke(very_long_prompt)
+except LlamaContextOverflowError as error:
+    print(error.n_prompt_tokens, error.n_ctx)
+```
+
+There are also `LlamaModelNotFoundError` (an unknown llama-swap entry),
+`LlamaInvalidRequestError`, `LlamaRateLimitError`, `LlamaAPIError` (5xx),
+`LlamaConnectionError` and `LlamaTimeoutError`.
+
+Connection errors, timeouts, 429 and 5xx are retried `max_retries` times
+(default 2) with exponential backoff, obeying `Retry-After`. `timeout` is in
+seconds and defaults to `None` (no timeout), since local generation can be slow.
+
+## Known limitations
+
+- **Forcing a tool with Qwen3.5/3.6.** With `tool_choice="required"` (also used
+  by `with_structured_output`'s default method), llama-server's grammar for these
+  models lets the model write text before the call and won't let it stop
+  without one. A model that doesn't want a tool keeps writing until
+  `max_tokens`, and there's no tool call. This is
+  [ggml-org/llama.cpp#27217](https://github.com/ggml-org/llama.cpp/issues/27217);
+  keep `max_tokens` set.
+- **A named `tool_choice`** is sent as `"required"` (see Tool calling).
+- **JSON schema bounds on numbers** (`minimum`, `maximum`, `multipleOf` on
+  `number`) aren't enforced by llama-server's grammar. Integer bounds are.
+- **Bare llama-server** (without llama-swap) isn't tested; `get_server_defaults`
+  needs llama-swap's `/upstream/<model>/props`.
+
+## Development
+
+Bug reports and pull requests are welcome on
+[GitHub](https://github.com/bharat029/langchain-llama-cpp/issues).
+
+```bash
+uv sync
+uv run pytest tests/unit_tests          # offline
+uv run ruff check && uv run ruff format --check && uv run pyright
+```
+
+The integration tests (LangChain's standard `langchain-tests` suite, on gemma,
+qwen and gpt-oss) need a llama-swap server. Copy `.env.example` to `.env` and
+set `LLAMA_BASE_URL`, then:
+
+```bash
+uv run --env-file .env pytest tests/integration_tests
+```
+
+The JSON llama-server sends back is described in
+`src/langchain_llama_cpp/_wire.py`.
+
+## License
+
+MIT

langchain_llama_cpp-0.1.0/pyproject.toml

@@ -0,0 +1,112 @@
+[project]
+name = "langchain-llama-cpp"
+version = "0.1.0"
+description = "An integration package connecting llama.cpp's llama-server and LangChain, with reasoning, timings, structured output and tool calling"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
+keywords = [
+    "langchain",
+    "llama.cpp",
+    "llama-server",
+    "llama-swap",
+    "llm",
+    "local-llm",
+    "chat-model",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Typing :: Typed",
+]
+dependencies = [
+    "httpx>=0.27,<1",
+    "langchain-core>=1.6,<2",
+]
+
+[[project.authors]]
+name = "Bharathan Mudaliar"
+email = "bharathanmudaliar@gmail.com"
+
+[project.urls]
+Repository = "https://github.com/bharat029/langchain-llama-cpp"
+Issues = "https://github.com/bharat029/langchain-llama-cpp/issues"
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "langchain-tests>=1.1.9",
+    "pyright>=1.1.414",
+    "pytest>=9.1.1",
+    "pytest-asyncio>=1.4.0",
+    "ruff>=0.17.0",
+]
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = 100
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "CPY001",
+    "COM812",
+    "ISC001",
+    "ANN401",
+    "TC002",
+    "TC003",
+    "RUF012",
+    "PLR0913",
+    "C901",
+    "PLR0912",
+    "PLR0915",
+    "PERF203",
+    "FIX002",
+    "TD002",
+    "TD003",
+]
+unfixable = ["B028"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+ignore-var-parameters = true
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.lint.extend-per-file-ignores]
+"tests/**/*.py" = [
+    "D",
+    "INP001",
+    "S101",
+    "PLR2004",
+    "SLF001",
+    "ARG001",
+    "PT011",
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.pyright]
+include = [
+    "src",
+    "tests",
+]
+pythonVersion = "3.10"
+venvPath = "."
+venv = ".venv"
+typeCheckingMode = "standard"
+strict = ["src"]