langchain-llamay 0.1.0__tar.gz

langchain_llamay-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+dist/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.benchmarks/
+uv.lock

langchain_llamay-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kosa8 Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_llamay-0.1.0/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.5
+Name: langchain-llamay
+Version: 0.1.0
+Summary: LangChain integration for llamay, a local LLM runtime
+Project-URL: Homepage, https://llamay.com
+Project-URL: Documentation, https://github.com/kosa8ai/langchain-llamay#readme
+Project-URL: Source, https://github.com/kosa8ai/langchain-llamay
+Author: Kosa8 Labs
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: langchain-core<2.0.0,>=1.0.0
+Requires-Dist: langchain-ollama<2.0.0,>=1.0.0
+Description-Content-Type: text/markdown
+
+# langchain-llamay
+
+LangChain integration for [llamay](https://llamay.com), a local LLM runtime for
+macOS, Windows and Linux.
+
+```bash
+pip install -U langchain-llamay
+```
+
+## Start a server
+
+Download llamay from [llamay.com/download](https://llamay.com/download/), then:
+
+```bash
+llamay serve -m qwen2.5:7b
+```
+
+llamay downloads a model it does not have yet, including GGUF repositories on
+Hugging Face (`hf.co/<owner>/<repo>:<quant>`), and listens on
+`http://localhost:11435`. Set `LLAMAY_HOST` to reach a server elsewhere.
+
+## Chat
+
+```python
+from langchain_llamay import ChatLlamay
+
+llm = ChatLlamay(model="qwen2.5:7b")
+print(llm.invoke("Why is the sky blue?").content)
+```
+
+Tool calling, structured output and streaming work as they do in
+`langchain-ollama`, which this package builds on: llamay serves Ollama's API.
+
+One thing Ollama lacks does work here: `tool_choice`. llamay constrains the
+answer to a call when one is required, so even a small model calls the tool it
+is told to.
+
+```python
+def get_weather(city: str) -> str:
+    """Get the weather for a city."""
+    return "sunny"
+
+
+llm.bind_tools([get_weather], tool_choice="get_weather").invoke("Hello!").tool_calls
+# [{'name': 'get_weather', 'args': {'city': ...}, ...}]
+```
+
+`tool_choice` takes `"auto"`, `"none"`, `"any"` or `True` (call one of the
+tools), a tool's name, or an OpenAI or Anthropic tool_choice object.
+
+```python
+from pydantic import BaseModel
+
+
+class City(BaseModel):
+    name: str
+    country: str
+
+
+llm.with_structured_output(City).invoke("Name a city in Japan.")
+```
+
+## Embeddings
+
+Start the server with an encoder beside the chat model:
+
+```bash
+llamay serve -m qwen2.5:7b -embed all-minilm:latest
+```
+
+```python
+from langchain_llamay import LlamayEmbeddings
+
+embeddings = LlamayEmbeddings(model="all-minilm:latest")
+vector = embeddings.embed_query("hello")
+```
+
+## Tests
+
+```bash
+uv run --group test pytest tests/unit_tests
+# with `llamay serve -m qwen2.5:0.5b -embed all-minilm:latest` running:
+uv run --group test pytest tests/integration_tests
+```

langchain_llamay-0.1.0/README.md

@@ -0,0 +1,84 @@
+# langchain-llamay
+
+LangChain integration for [llamay](https://llamay.com), a local LLM runtime for
+macOS, Windows and Linux.
+
+```bash
+pip install -U langchain-llamay
+```
+
+## Start a server
+
+Download llamay from [llamay.com/download](https://llamay.com/download/), then:
+
+```bash
+llamay serve -m qwen2.5:7b
+```
+
+llamay downloads a model it does not have yet, including GGUF repositories on
+Hugging Face (`hf.co/<owner>/<repo>:<quant>`), and listens on
+`http://localhost:11435`. Set `LLAMAY_HOST` to reach a server elsewhere.
+
+## Chat
+
+```python
+from langchain_llamay import ChatLlamay
+
+llm = ChatLlamay(model="qwen2.5:7b")
+print(llm.invoke("Why is the sky blue?").content)
+```
+
+Tool calling, structured output and streaming work as they do in
+`langchain-ollama`, which this package builds on: llamay serves Ollama's API.
+
+One thing Ollama lacks does work here: `tool_choice`. llamay constrains the
+answer to a call when one is required, so even a small model calls the tool it
+is told to.
+
+```python
+def get_weather(city: str) -> str:
+    """Get the weather for a city."""
+    return "sunny"
+
+
+llm.bind_tools([get_weather], tool_choice="get_weather").invoke("Hello!").tool_calls
+# [{'name': 'get_weather', 'args': {'city': ...}, ...}]
+```
+
+`tool_choice` takes `"auto"`, `"none"`, `"any"` or `True` (call one of the
+tools), a tool's name, or an OpenAI or Anthropic tool_choice object.
+
+```python
+from pydantic import BaseModel
+
+
+class City(BaseModel):
+    name: str
+    country: str
+
+
+llm.with_structured_output(City).invoke("Name a city in Japan.")
+```
+
+## Embeddings
+
+Start the server with an encoder beside the chat model:
+
+```bash
+llamay serve -m qwen2.5:7b -embed all-minilm:latest
+```
+
+```python
+from langchain_llamay import LlamayEmbeddings
+
+embeddings = LlamayEmbeddings(model="all-minilm:latest")
+vector = embeddings.embed_query("hello")
+```
+
+## Tests
+
+```bash
+uv run --group test pytest tests/unit_tests
+# with `llamay serve -m qwen2.5:0.5b -embed all-minilm:latest` running:
+uv run --group test pytest tests/integration_tests
+```

langchain_llamay-0.1.0/langchain_llamay/__init__.py

@@ -0,0 +1,13 @@
+"""LangChain integration for llamay, a local LLM runtime."""
+
+from importlib import metadata
+
+from langchain_llamay.chat_models import ChatLlamay
+from langchain_llamay.embeddings import LlamayEmbeddings
+
+try:
+    __version__ = metadata.version(__package__)
+except metadata.PackageNotFoundError:
+    __version__ = ""
+
+__all__ = ["ChatLlamay", "LlamayEmbeddings", "__version__"]

langchain_llamay-0.1.0/langchain_llamay/_base.py

@@ -0,0 +1,14 @@
+"""Where a llamay server is, when nobody says."""
+
+import os
+
+DEFAULT_BASE_URL = "http://localhost:11435"
+
+
+def default_base_url() -> str:
+    """LLAMAY_HOST, or llamay's own port on this machine.
+
+    llamay listens on 11435, beside Ollama's 11434, so the two can run at once;
+    pointing at 11434 by default would quietly talk to Ollama instead.
+    """
+    return os.environ.get("LLAMAY_HOST") or DEFAULT_BASE_URL

langchain_llamay-0.1.0/langchain_llamay/_client.py

@@ -0,0 +1,52 @@
+"""The ollama client, able to send llamay's tool_choice.
+
+Ollama's /api/chat has no tool_choice, so the ollama client has no argument for
+it. llamay's does: it reads the field in OpenAI's and Anthropic's shapes and
+constrains generation to a call, which is how a small local model can be made
+to call a tool at all. These subclasses add the field to the request body.
+
+The choice travels in a context variable rather than on the client, because one
+client serves every call a chat model makes, concurrent ones included. The body
+is built inside chat() before it returns -- for a stream too, whose generator
+only reads the response -- so the variable is still set when _request runs.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import Any
+
+from ollama import AsyncClient, Client
+
+_tool_choice: ContextVar[Any] = ContextVar("llamay_tool_choice", default=None)
+
+
+def _with_choice(kwargs: dict[str, Any], args: tuple[Any, ...]) -> dict[str, Any]:
+    choice = _tool_choice.get()
+    if choice is not None and len(args) >= 2 and args[1] == "/api/chat" and "json" in kwargs:
+        kwargs = {**kwargs, "json": {**kwargs["json"], "tool_choice": choice}}
+    return kwargs
+
+
+class LlamayClient(Client):
+    def chat(self, *args: Any, tool_choice: Any = None, **kwargs: Any) -> Any:
+        token = _tool_choice.set(tool_choice)
+        try:
+            return super().chat(*args, **kwargs)
+        finally:
+            _tool_choice.reset(token)
+
+    def _request(self, cls: Any, *args: Any, **kwargs: Any) -> Any:
+        return super()._request(cls, *args, **_with_choice(kwargs, args))
+
+
+class LlamayAsyncClient(AsyncClient):
+    async def chat(self, *args: Any, tool_choice: Any = None, **kwargs: Any) -> Any:
+        token = _tool_choice.set(tool_choice)
+        try:
+            return await super().chat(*args, **kwargs)
+        finally:
+            _tool_choice.reset(token)
+
+    async def _request(self, cls: Any, *args: Any, **kwargs: Any) -> Any:
+        return await super()._request(cls, *args, **_with_choice(kwargs, args))

langchain_llamay-0.1.0/langchain_llamay/chat_models.py

@@ -0,0 +1,131 @@
+"""llamay chat models."""
+
+from collections.abc import Callable, Sequence
+from typing import Any, Literal
+
+from langchain_core.language_models import LangSmithParams, LanguageModelInput
+from langchain_core.messages import AIMessage, BaseMessage
+from langchain_core.runnables import Runnable
+from langchain_core.tools import BaseTool
+from langchain_core.utils.function_calling import convert_to_openai_tool
+from langchain_ollama import ChatOllama
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from langchain_llamay._base import default_base_url
+from langchain_llamay._client import LlamayAsyncClient, LlamayClient
+
+
+def _llamay_tool_choice(choice: dict | str | bool | None) -> dict | str | None:
+    """LangChain's tool_choice in a form llamay reads.
+
+    llamay takes OpenAI's strings and object and Anthropic's object. LangChain
+    adds True for "call something" and a bare tool name for "call this one".
+    """
+    if choice is None or choice is False:
+        return None
+    if choice is True:
+        return "required"
+    if isinstance(choice, dict):
+        return choice
+    if choice in ("auto", "none", "any", "required"):
+        return choice
+    return {"type": "function", "function": {"name": choice}}
+
+
+class ChatLlamay(ChatOllama):
+    """Chat with a model served by `llamay serve`.
+
+    llamay serves Ollama's API, so this is `ChatOllama` pointed at llamay, with
+    one addition Ollama does not have: `tool_choice` is honoured. llamay
+    constrains generation to a call when one is required, so even a small
+    local model calls the tool it is told to.
+
+    Setup:
+        Install llamay from https://llamay.com/download/ and start a server:
+
+        .. code-block:: bash
+
+            llamay serve -m qwen2.5:7b
+            pip install -U langchain-llamay
+
+    Instantiate:
+        .. code-block:: python
+
+            from langchain_llamay import ChatLlamay
+
+            llm = ChatLlamay(model="qwen2.5:7b")
+            llm.invoke("Why is the sky blue?")
+
+    Tool calling:
+        .. code-block:: python
+
+            llm.bind_tools([get_weather], tool_choice="get_weather").invoke("Hi")
+
+    Key init args:
+        model: str
+            Any name `llamay models` lists, or a Hugging Face reference such as
+            ``hf.co/bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M``.
+        base_url: str | None
+            The server. Defaults to ``LLAMAY_HOST``, else
+            ``http://localhost:11435``.
+    """
+
+    base_url: str | None = Field(default_factory=default_base_url)
+
+    @model_validator(mode="after")
+    def _llamay_clients(self) -> Self:
+        # ChatOllama has built its clients by now, with every setting it was
+        # given; giving them the subclass that can send tool_choice keeps all
+        # of that rather than rebuilding it here.
+        if self._client is not None:
+            self._client.__class__ = LlamayClient
+        if self._async_client is not None:
+            self._async_client.__class__ = LlamayAsyncClient
+        return self
+
+    @property
+    def _llm_type(self) -> str:
+        return "chat-llamay"
+
+    def _get_ls_params(self, stop: list[str] | None = None, **kwargs: Any) -> LangSmithParams:
+        params = super()._get_ls_params(stop=stop, **kwargs)
+        params["ls_provider"] = "llamay"
+        # A per-call model= is the model that answered; ChatOllama reports the
+        # constructor's, which files the trace under the wrong model.
+        params["ls_model_name"] = kwargs.get("model", self.model)
+        return params
+
+    def bind_tools(
+        self,
+        tools: Sequence[dict[str, Any] | type | Callable | BaseTool],
+        *,
+        tool_choice: dict | str | Literal["auto", "any"] | bool | None = None,  # noqa: PYI051
+        **kwargs: Any,
+    ) -> Runnable[LanguageModelInput, AIMessage]:
+        """Bind tools, and optionally which of them the model must call.
+
+        Args:
+            tools: Tool definitions, in any form ``convert_to_openai_tool`` reads.
+            tool_choice: ``"auto"``, ``"none"``, ``"any"`` or ``True`` (call one
+                of them), a tool's name (call that one), or an OpenAI or
+                Anthropic tool_choice object.
+            kwargs: Passed to ``self.bind``.
+        """
+        choice = _llamay_tool_choice(tool_choice)
+        if choice is not None:
+            kwargs["tool_choice"] = choice
+        # Not super().bind_tools: its own tool_choice parameter would take the
+        # choice and drop it, which is what this method exists to stop.
+        return self.bind(tools=[convert_to_openai_tool(t) for t in tools], **kwargs)
+
+    def _chat_params(
+        self, messages: list[BaseMessage], stop: list[str] | None = None, **kwargs: Any
+    ) -> dict[str, Any]:
+        # Kept out of ChatOllama's params, which go to the ollama client as
+        # they are, and put back for LlamayClient.chat to send.
+        choice = kwargs.pop("tool_choice", None)
+        params = super()._chat_params(messages, stop, **kwargs)
+        if choice is not None:
+            params["tool_choice"] = choice
+        return params

langchain_llamay-0.1.0/langchain_llamay/embeddings.py

@@ -0,0 +1,26 @@
+"""llamay embeddings."""
+
+from langchain_ollama import OllamaEmbeddings
+from pydantic import Field
+
+from langchain_llamay._base import default_base_url
+
+
+class LlamayEmbeddings(OllamaEmbeddings):
+    """Embeddings from the encoder a llamay server was started with.
+
+    llamay serves embeddings from an encoder loaded beside the chat model:
+
+    .. code-block:: bash
+
+        llamay serve -m qwen2.5:7b -embed all-minilm:latest
+
+    .. code-block:: python
+
+        from langchain_llamay import LlamayEmbeddings
+
+        embeddings = LlamayEmbeddings(model="all-minilm:latest")
+        embeddings.embed_query("hello")
+    """
+
+    base_url: str | None = Field(default_factory=default_base_url)

langchain_llamay-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "langchain-llamay"
+version = "0.1.0"
+description = "LangChain integration for llamay, a local LLM runtime"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Kosa8 Labs" }]
+requires-python = ">=3.10"
+dependencies = [
+    "langchain-core>=1.0.0,<2.0.0",
+    "langchain-ollama>=1.0.0,<2.0.0",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://llamay.com"
+Documentation = "https://github.com/kosa8ai/langchain-llamay#readme"
+Source = "https://github.com/kosa8ai/langchain-llamay"
+
+[dependency-groups]
+test = ["pytest>=8,<9", "pytest-asyncio>=0.23", "langchain-tests>=1.0.0,<2.0.0"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+markers = ["compile: placeholder test that only checks the package imports"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["langchain_llamay"]

langchain_llamay-0.1.0/tests/integration_tests/test_chat_models.py

@@ -0,0 +1,31 @@
+"""Standard tests against a live `llamay serve -m qwen2.5:0.5b -embed all-minilm:latest`."""
+
+import os
+
+from langchain_tests.integration_tests import ChatModelIntegrationTests
+
+from langchain_llamay import ChatLlamay
+
+MODEL = os.environ.get("LLAMAY_TEST_MODEL", "qwen2.5:0.5b")
+
+
+class TestChatLlamayIntegration(ChatModelIntegrationTests):
+    @property
+    def chat_model_class(self) -> type[ChatLlamay]:
+        return ChatLlamay
+
+    @property
+    def chat_model_params(self) -> dict:
+        return {"model": MODEL, "temperature": 0}
+
+    @property
+    def has_tool_calling(self) -> bool:
+        return True
+
+    @property
+    def has_tool_choice(self) -> bool:
+        return True
+
+    @property
+    def supports_image_inputs(self) -> bool:
+        return False

langchain_llamay-0.1.0/tests/integration_tests/test_embeddings.py

@@ -0,0 +1,13 @@
+from langchain_tests.integration_tests import EmbeddingsIntegrationTests
+
+from langchain_llamay import LlamayEmbeddings
+
+
+class TestLlamayEmbeddingsIntegration(EmbeddingsIntegrationTests):
+    @property
+    def embeddings_class(self) -> type[LlamayEmbeddings]:
+        return LlamayEmbeddings
+
+    @property
+    def embedding_model_params(self) -> dict:
+        return {"model": "all-minilm:latest"}

langchain_llamay-0.1.0/tests/unit_tests/test_chat_models.py

@@ -0,0 +1,69 @@
+from langchain_tests.unit_tests import ChatModelUnitTests
+
+from langchain_llamay import ChatLlamay
+from langchain_llamay._base import DEFAULT_BASE_URL
+
+
+class TestChatLlamayUnit(ChatModelUnitTests):
+    @property
+    def chat_model_class(self) -> type[ChatLlamay]:
+        return ChatLlamay
+
+    @property
+    def chat_model_params(self) -> dict:
+        return {"model": "qwen2.5:0.5b"}
+
+
+def test_defaults_to_llamays_port(monkeypatch) -> None:
+    monkeypatch.delenv("LLAMAY_HOST", raising=False)
+    assert ChatLlamay(model="qwen2.5:0.5b").base_url == DEFAULT_BASE_URL == "http://localhost:11435"
+
+
+def test_reads_llamay_host(monkeypatch) -> None:
+    monkeypatch.setenv("LLAMAY_HOST", "http://gpu-box:11435")
+    assert ChatLlamay(model="qwen2.5:0.5b").base_url == "http://gpu-box:11435"
+
+
+def test_an_explicit_base_url_wins(monkeypatch) -> None:
+    monkeypatch.setenv("LLAMAY_HOST", "http://gpu-box:11435")
+    assert ChatLlamay(model="m", base_url="http://other:1").base_url == "http://other:1"
+
+
+def test_traces_as_llamay() -> None:
+    llm = ChatLlamay(model="qwen2.5:0.5b")
+    assert llm._llm_type == "chat-llamay"
+    assert llm._get_ls_params()["ls_provider"] == "llamay"
+
+
+def test_tool_choice_reaches_the_request(monkeypatch) -> None:
+    # ChatOllama drops tool_choice; ChatLlamay sends it, in the shape llamay
+    # reads, on the same /api/chat request as the tools.
+    sent: list[dict] = []
+
+    def fake_request(self, cls, *args, **kwargs):
+        sent.append(kwargs.get("json", {}))
+        return cls(model="m", message={"role": "assistant", "content": ""}, done=True)
+
+    import ollama
+
+    monkeypatch.setattr(ollama.Client, "_request", fake_request)
+
+    def get_weather(city: str) -> str:
+        """Get the weather for a city."""
+        return "sunny"
+
+    llm = ChatLlamay(model="qwen2.5:0.5b")
+    for choice, want in [
+        ("get_weather", {"type": "function", "function": {"name": "get_weather"}}),
+        ("any", "any"),
+        (True, "required"),
+        ("auto", "auto"),
+    ]:
+        sent.clear()
+        llm.bind_tools([get_weather], tool_choice=choice).invoke("hi", stream=False)
+        assert sent and sent[-1].get("tool_choice") == want, (choice, sent)
+        assert sent[-1]["tools"][0]["function"]["name"] == "get_weather"
+
+    sent.clear()
+    llm.bind_tools([get_weather]).invoke("hi", stream=False)
+    assert "tool_choice" not in sent[-1]

langchain_llamay-0.1.0/tests/unit_tests/test_embeddings.py

@@ -0,0 +1,18 @@
+from langchain_tests.unit_tests import EmbeddingsUnitTests
+
+from langchain_llamay import LlamayEmbeddings
+
+
+class TestLlamayEmbeddingsUnit(EmbeddingsUnitTests):
+    @property
+    def embeddings_class(self) -> type[LlamayEmbeddings]:
+        return LlamayEmbeddings
+
+    @property
+    def embedding_model_params(self) -> dict:
+        return {"model": "all-minilm:latest"}
+
+
+def test_embeddings_default_to_llamays_port(monkeypatch) -> None:
+    monkeypatch.delenv("LLAMAY_HOST", raising=False)
+    assert LlamayEmbeddings(model="all-minilm:latest").base_url == "http://localhost:11435"