llama-cpp-haystack 0.4.4__tar.gz → 1.1.0__tar.gz

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.1.0}/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [integrations/llama_cpp-v1.0.0] - 2025-02-07
+
+### 🚀 Features
+
+- [**breaking**] Llama.cpp - unified support for tools + refactoring (#1357)
+
+
+## [integrations/llama_cpp-v0.4.4] - 2025-01-16
+
+### 🧹 Chores
+
+- Llama.cpp - gently handle the removal of ChatMessage.from_function (#1298)
+
+
 ## [integrations/llama_cpp-v0.4.3] - 2024-12-19
 
 ### 🐛 Bug Fixes

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llama-cpp-haystack
-Version: 0.4.4
+Version: 1.1.0
 Summary: An integration between the llama.cpp LLM framework and Haystack
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/llama_cpp#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -12,15 +12,15 @@ License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.8
-Requires-Dist: haystack-ai>=2.9.0
+Requires-Python: >=3.9
+Requires-Dist: haystack-ai>=2.13.0
 Requires-Dist: llama-cpp-python>=0.2.87
 Description-Content-Type: text/markdown
 

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.1.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "llama-cpp-haystack"
 dynamic = ["version"]
 description = 'An integration between the llama.cpp LLM framework and Haystack'
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = "Apache-2.0"
 keywords = []
 authors = [
@@ -18,15 +18,15 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Development Status :: 4 - Beta",
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai>=2.9.0", "llama-cpp-python>=0.2.87"]
+dependencies = ["haystack-ai>=2.13.0", "llama-cpp-python>=0.2.87"]
 
 [project.urls]
 Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/llama_cpp#readme"
@@ -46,35 +46,35 @@ git_describe_command = 'git describe --tags --match="integrations/llama_cpp-v[0-
 
 [tool.hatch.envs.default]
 installer = "uv"
+dependencies = ["haystack-pydoc-tools", "ruff"]
+[tool.hatch.envs.default.scripts]
+docs = ["pydoc-markdown pydoc/config.yml"]
+fmt = "ruff check --fix {args} && ruff format {args}"
+fmt-check = "ruff check {args} && ruff format --check {args}"
+
+[tool.hatch.envs.test]
 dependencies = [
-    "coverage[toml]>=6.5",
     "pytest",
+    "pytest-asyncio",
+    "pytest-cov",
     "pytest-rerunfailures",
-    "haystack-pydoc-tools",
-    "transformers[sentencepiece]",
+    "mypy",
+    "pip",
+    "transformers[sentencepiece]"
 ]
-[tool.hatch.envs.default.scripts]
-test = "pytest {args:tests}"
-test-cov = "coverage run -m pytest {args:tests}"
-test-cov-retry = "test-cov --reruns 3 --reruns-delay 30 -x"
-cov-report = ["- coverage combine", "coverage report"]
-cov = ["test-cov", "cov-report"]
-cov-retry = ["test-cov-retry", "cov-report"]
-docs = ["pydoc-markdown pydoc/config.yml"]
-[[tool.hatch.envs.all.matrix]]
-python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
 
-[tool.hatch.envs.lint]
-installer = "uv"
-detached = true
-dependencies = ["pip", "black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
+[tool.hatch.envs.test.scripts]
+unit = 'pytest -m "not integration" {args:tests}'
+integration = 'pytest -m "integration" {args:tests}'
+all = 'pytest {args:tests}'
+cov-retry = 'all --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x'
+types = "mypy -p haystack_integrations.components.generators.llama_cpp {args}"
 
-[tool.hatch.envs.lint.scripts]
-typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = ["ruff check {args:.}", "black --check --diff {args:.}"]
-fmt = ["black {args:.}", "ruff check --fix {args:.}", "style"]
-all = ["style", "typing"]
+[tool.mypy]
+install_types = true
+non_interactive = true
+check_untyped_defs = true
+disallow_incomplete_defs = true
 
 [tool.hatch.metadata]
 allow-direct-references = true
@@ -164,7 +164,3 @@ markers = [
     "integration: marks tests as slow (deselect with '-m \"not integration\"')",
 ]
 addopts = ["--import-mode=importlib"]
-
-[[tool.mypy.overrides]]
-module = ["haystack.*", "haystack_integrations.*", "pytest.*", "llama_cpp.*"]
-ignore_missing_imports = true

llama_cpp_haystack-1.1.0/src/haystack_integrations/components/generators/llama_cpp/chat/chat_generator.py

@@ -0,0 +1,294 @@
+import json
+from typing import Any, Dict, List, Optional, Union
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.dataclasses import ChatMessage, ToolCall
+from haystack.tools import (
+    Tool,
+    Toolset,
+    _check_duplicate_tool_names,
+    deserialize_tools_or_toolset_inplace,
+    serialize_tools_or_toolset,
+)
+from llama_cpp import (
+    ChatCompletionMessageToolCall,
+    ChatCompletionRequestAssistantMessage,
+    ChatCompletionRequestMessage,
+    ChatCompletionResponseChoice,
+    ChatCompletionTool,
+    CreateChatCompletionResponse,
+    Llama,
+)
+from llama_cpp.llama_tokenizer import LlamaHFTokenizer
+
+logger = logging.getLogger(__name__)
+
+
+def _convert_message_to_llamacpp_format(message: ChatMessage) -> ChatCompletionRequestMessage:
+    """
+    Convert a ChatMessage to the format expected by llama.cpp Chat API.
+    """
+    text_contents = message.texts
+    tool_calls = message.tool_calls
+    tool_call_results = message.tool_call_results
+
+    if not text_contents and not tool_calls and not tool_call_results:
+        msg = "A `ChatMessage` must contain at least one `TextContent`, `ToolCall`, or `ToolCallResult`."
+        raise ValueError(msg)
+    elif len(text_contents) + len(tool_call_results) > 1:
+        msg = "A `ChatMessage` can only contain one `TextContent` or one `ToolCallResult`."
+        raise ValueError(msg)
+
+    role = message._role.value
+
+    if role == "tool" and tool_call_results:
+        if tool_call_results[0].origin.id is None:
+            msg = "`ToolCall` must have a non-null `id` attribute to be used with llama.cpp."
+            raise ValueError(msg)
+        return {
+            "role": "function",
+            "content": tool_call_results[0].result,
+            "name": tool_call_results[0].origin.tool_name,
+        }
+
+    if role == "system":
+        content = text_contents[0] if text_contents else None
+        return {"role": "system", "content": content}
+
+    if role == "user":
+        content = text_contents[0] if text_contents else None
+        return {"role": "user", "content": content}
+
+    if role == "assistant":
+        result: ChatCompletionRequestAssistantMessage = {"role": "assistant"}
+
+        if text_contents:
+            result["content"] = text_contents[0]
+
+        if tool_calls:
+            llamacpp_tool_calls: List[ChatCompletionMessageToolCall] = []
+            for tc in tool_calls:
+                if tc.id is None:
+                    msg = "`ToolCall` must have a non-null `id` attribute to be used with llama.cpp."
+                    raise ValueError(msg)
+                llamacpp_tool_calls.append(
+                    {
+                        "id": tc.id,
+                        "type": "function",
+                        # We disable ensure_ascii so special chars like emojis are not converted
+                        "function": {"name": tc.tool_name, "arguments": json.dumps(tc.arguments, ensure_ascii=False)},
+                    }
+                )
+            result["tool_calls"] = llamacpp_tool_calls
+
+        return result
+
+    error_msg = f"Unknown role: {role}"
+    raise ValueError(error_msg)
+
+
+@component
+class LlamaCppChatGenerator:
+    """
+    Provides an interface to generate text using LLM via llama.cpp.
+
+    [llama.cpp](https://github.com/ggerganov/llama.cpp) is a project written in C/C++ for efficient inference of LLMs.
+    It employs the quantized GGUF format, suitable for running these models on standard machines (even without GPUs).
+
+    Usage example:
+    ```python
+    from haystack_integrations.components.generators.llama_cpp import LlamaCppChatGenerator
+    user_message = [ChatMessage.from_user("Who is the best American actor?")]
+    generator = LlamaCppGenerator(model="zephyr-7b-beta.Q4_0.gguf", n_ctx=2048, n_batch=512)
+
+    print(generator.run(user_message, generation_kwargs={"max_tokens": 128}))
+    # {"replies": [ChatMessage(content="John Cusack", role=<ChatRole.ASSISTANT: "assistant">, name=None, meta={...}]}
+    ```
+    """
+
+    def __init__(
+        self,
+        model: str,
+        n_ctx: Optional[int] = 0,
+        n_batch: Optional[int] = 512,
+        model_kwargs: Optional[Dict[str, Any]] = None,
+        generation_kwargs: Optional[Dict[str, Any]] = None,
+        *,
+        tools: Optional[Union[List[Tool], Toolset]] = None,
+    ):
+        """
+        :param model: The path of a quantized model for text generation, for example, "zephyr-7b-beta.Q4_0.gguf".
+            If the model path is also specified in the `model_kwargs`, this parameter will be ignored.
+        :param n_ctx: The number of tokens in the context. When set to 0, the context will be taken from the model.
+        :param n_batch: Prompt processing maximum batch size.
+        :param model_kwargs: Dictionary containing keyword arguments used to initialize the LLM for text generation.
+            These keyword arguments provide fine-grained control over the model loading.
+            In case of duplication, these kwargs override `model`, `n_ctx`, and `n_batch` init parameters.
+            For more information on the available kwargs, see
+            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.__init__).
+        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
+            For more information on the available kwargs, see
+            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
+        :param tools:
+            A list of tools or a Toolset for which the model can prepare calls.
+            This parameter can accept either a list of `Tool` objects or a `Toolset` instance.
+        """
+
+        model_kwargs = model_kwargs or {}
+        generation_kwargs = generation_kwargs or {}
+
+        # check if the model_kwargs contain the essential parameters
+        # otherwise, populate them with values from init parameters
+        model_kwargs.setdefault("model_path", model)
+        model_kwargs.setdefault("n_ctx", n_ctx)
+        model_kwargs.setdefault("n_batch", n_batch)
+
+        _check_duplicate_tool_names(list(tools or []))
+
+        self.model_path = model
+        self.n_ctx = n_ctx
+        self.n_batch = n_batch
+        self.model_kwargs = model_kwargs
+        self.generation_kwargs = generation_kwargs
+        self._model: Optional[Llama] = None
+        self.tools = tools
+
+    def warm_up(self):
+        if "hf_tokenizer_path" in self.model_kwargs and "tokenizer" not in self.model_kwargs:
+            tokenizer = LlamaHFTokenizer.from_pretrained(self.model_kwargs["hf_tokenizer_path"])
+            self.model_kwargs["tokenizer"] = tokenizer
+
+        if self._model is None:
+            self._model = Llama(**self.model_kwargs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+              Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            model=self.model_path,
+            n_ctx=self.n_ctx,
+            n_batch=self.n_batch,
+            model_kwargs=self.model_kwargs,
+            generation_kwargs=self.generation_kwargs,
+            tools=serialize_tools_or_toolset(self.tools),
+        )
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "LlamaCppChatGenerator":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+        return default_from_dict(cls, data)
+
+    @component.output_types(replies=List[ChatMessage])
+    def run(
+        self,
+        messages: List[ChatMessage],
+        generation_kwargs: Optional[Dict[str, Any]] = None,
+        *,
+        tools: Optional[Union[List[Tool], Toolset]] = None,
+    ) -> Dict[str, List[ChatMessage]]:
+        """
+        Run the text generation model on the given list of ChatMessages.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages.
+        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
+            For more information on the available kwargs, see
+            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
+        :param tools:
+            A list of tools or a Toolset for which the model can prepare calls. If set, it will override the `tools`
+            parameter set during component initialization.
+        :returns: A dictionary with the following keys:
+            - `replies`: The responses from the model
+        """
+        if self._model is None:
+            error_msg = "The model has not been loaded. Please call warm_up() before running."
+            raise RuntimeError(error_msg)
+
+        if not messages:
+            return {"replies": []}
+
+        updated_generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
+        formatted_messages = [_convert_message_to_llamacpp_format(msg) for msg in messages]
+
+        tools = tools or self.tools
+        if isinstance(tools, Toolset):
+            tools = list(tools)
+        _check_duplicate_tool_names(tools)
+
+        llamacpp_tools: List[ChatCompletionTool] = []
+        if tools:
+            for t in tools:
+                llamacpp_tools.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": t.tool_spec["name"],
+                            "description": t.tool_spec.get("description", ""),
+                            "parameters": t.tool_spec.get("parameters", {}),
+                        },
+                    }
+                )
+
+        response = self._model.create_chat_completion(
+            messages=formatted_messages, tools=llamacpp_tools, **updated_generation_kwargs
+        )
+
+        replies = []
+        if not isinstance(response, dict):
+            msg = f"Expected a dictionary response, got a different object: {response}"
+            raise ValueError(msg)
+
+        for choice in response["choices"]:
+            chat_message = self._convert_chat_completion_choice_to_chat_message(choice, response)
+            replies.append(chat_message)
+
+        return {"replies": replies}
+
+    @staticmethod
+    def _convert_chat_completion_choice_to_chat_message(
+        choice: ChatCompletionResponseChoice, response: CreateChatCompletionResponse
+    ) -> ChatMessage:
+        llamacpp_message = choice["message"]
+        text_content = llamacpp_message.get("content", "") or None
+        tool_calls = []
+
+        if llamacpp_tool_calls := llamacpp_message.get("tool_calls", []):
+            for llamacpp_tc in llamacpp_tool_calls:
+                arguments_str = llamacpp_tc["function"]["arguments"]
+                try:
+                    arguments = json.loads(arguments_str)
+                    tool_calls.append(
+                        ToolCall(id=llamacpp_tc["id"], tool_name=llamacpp_tc["function"]["name"], arguments=arguments)
+                    )
+                except json.JSONDecodeError:
+                    logger.warning(
+                        "Llama.cpp returned a malformed JSON string for tool call arguments. This tool call "
+                        "will be skipped. Tool call ID: {tc_id}, Tool name: {tc_name}, Arguments: {tc_args}",
+                        tc_id=llamacpp_tc["id"],
+                        tc_name=llamacpp_tc["function"]["name"],
+                        tc_args=arguments_str,
+                    )
+
+        meta = {
+            "response_id": response["id"],
+            "model": response["model"],
+            "created": response["created"],
+            "index": choice["index"],
+            "finish_reason": choice["finish_reason"],
+            "usage": response["usage"],
+        }
+
+        return ChatMessage.from_assistant(text=text_content, tool_calls=tool_calls, meta=meta)

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.1.0}/src/haystack_integrations/components/generators/llama_cpp/generator.py

@@ -1,7 +1,6 @@
-import logging
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
-from haystack import component
+from haystack import component, logging
 
 from llama_cpp import Llama
 
@@ -63,14 +62,16 @@ class LlamaCppGenerator:
         self.n_batch = n_batch
         self.model_kwargs = model_kwargs
         self.generation_kwargs = generation_kwargs
-        self.model = None
+        self.model: Optional[Llama] = None
 
     def warm_up(self):
         if self.model is None:
             self.model = Llama(**self.model_kwargs)
 
     @component.output_types(replies=List[str], meta=List[Dict[str, Any]])
-    def run(self, prompt: str, generation_kwargs: Optional[Dict[str, Any]] = None):
+    def run(
+        self, prompt: str, generation_kwargs: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Union[List[str], List[Dict[str, Any]]]]:
         """
         Run the text generation model on the given prompt.
 
@@ -93,6 +94,10 @@ class LlamaCppGenerator:
         updated_generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
 
         output = self.model.create_completion(prompt=prompt, **updated_generation_kwargs)
+        if not isinstance(output, dict):
+            msg = f"Expected a dictionary response, got a different object: {output}"
+            raise ValueError(msg)
+
         replies = [output["choices"][0]["text"]]
 
-        return {"replies": replies, "meta": [output]}
+        return {"replies": replies, "meta": [dict(output.items())]}

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.1.0}/tests/test_chat_generator.py

@@ -2,14 +2,16 @@ import json
 import os
 import urllib.request
 from pathlib import Path
+from typing import Annotated
 from unittest.mock import MagicMock
 
 import pytest
 from haystack import Document, Pipeline
 from haystack.components.builders import ChatPromptBuilder
 from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
-from haystack.dataclasses import ChatMessage, ChatRole
+from haystack.dataclasses import ChatMessage, ChatRole, TextContent, ToolCall
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.tools import Tool, Toolset, create_tool_from_function
 
 from haystack_integrations.components.generators.llama_cpp.chat.chat_generator import (
     LlamaCppChatGenerator,
@@ -22,6 +24,24 @@ def model_path():
     return Path(__file__).parent / "models"
 
 
+def get_current_temperature(location: Annotated[str, "The city and state, e.g. San Francisco, CA"]):
+    """Get the current temperature in a given location"""
+
+    if "tokyo" in location.lower():
+        return {"location": "Tokyo", "temperature": "10", "unit": "celsius"}
+    if "san francisco" in location.lower():
+        return {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}
+    if "paris" in location.lower():
+        return {"location": "Paris", "temperature": "22", "unit": "celsius"}
+
+    return {"location": location, "temperature": "unknown"}
+
+
+@pytest.fixture
+def temperature_tool():
+    return create_tool_from_function(get_current_temperature)
+
+
 def download_file(file_link, filename, capsys):
     # Checks if the file already exists before downloading
     if not os.path.isfile(filename):
@@ -35,17 +55,72 @@ def download_file(file_link, filename, capsys):
 
 def test_convert_message_to_llamacpp_format():
     message = ChatMessage.from_system("You are good assistant")
-    assert _convert_message_to_llamacpp_format(message) == {"role": "system", "content": "You are good assistant"}
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "system",
+        "content": "You are good assistant",
+    }
 
     message = ChatMessage.from_user("I have a question")
-    assert _convert_message_to_llamacpp_format(message) == {"role": "user", "content": "I have a question"}
-
-    if hasattr(ChatMessage, "from_function"):
-        message = ChatMessage.from_function("Function call", "function_name")
-        converted_message = _convert_message_to_llamacpp_format(message)
-        assert converted_message["role"] in ("function", "tool")
-        assert converted_message["name"] == "function_name"
-        assert converted_message["content"] == "Function call"
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "user",
+        "content": "I have a question",
+    }
+
+    message = ChatMessage.from_assistant(text="I have an answer", meta={"finish_reason": "stop"})
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "assistant",
+        "content": "I have an answer",
+    }
+
+    message = ChatMessage.from_assistant(
+        tool_calls=[ToolCall(id="123", tool_name="weather", arguments={"city": "Paris"})]
+    )
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "assistant",
+        "tool_calls": [
+            {
+                "type": "function",
+                "function": {"name": "weather", "arguments": '{"city": "Paris"}'},
+                "id": "123",
+            }
+        ],
+    }
+
+    tool_result = json.dumps({"weather": "sunny", "temperature": "25"})
+    message = ChatMessage.from_tool(
+        tool_result=tool_result,
+        origin=ToolCall(id="123", tool_name="weather", arguments={"city": "Paris"}),
+    )
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "function",
+        "content": tool_result,
+        "name": "weather",
+    }
+
+
+def test_convert_message_to_llamacpp_invalid():
+    message = ChatMessage(_role=ChatRole.ASSISTANT, _content=[])
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
+
+    message = ChatMessage(
+        _role=ChatRole.ASSISTANT,
+        _content=[
+            TextContent(text="I have an answer"),
+            TextContent(text="I have another answer"),
+        ],
+    )
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
+
+    tool_call_null_id = ToolCall(id=None, tool_name="weather", arguments={"city": "Paris"})
+    message = ChatMessage.from_assistant(tool_calls=[tool_call_null_id])
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
+
+    message = ChatMessage.from_tool(tool_result="result", origin=tool_call_null_id)
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
 
 
 class TestLlamaCppChatGenerator:
@@ -68,7 +143,7 @@ class TestLlamaCppChatGenerator:
     def generator_mock(self):
         mock_model = MagicMock()
         generator = LlamaCppChatGenerator(model="test_model.gguf", n_ctx=2048, n_batch=512)
-        generator.model = mock_model
+        generator._model = mock_model
         return generator, mock_model
 
     def test_default_init(self):
@@ -99,6 +174,63 @@ class TestLlamaCppChatGenerator:
         assert generator.model_kwargs == {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512}
         assert generator.generation_kwargs == {}
 
+    def test_init_with_toolset(self, temperature_tool):
+        toolset = Toolset([temperature_tool])
+        generator = LlamaCppChatGenerator(model="test_model.gguf", tools=toolset)
+        assert generator.tools == toolset
+
+    def test_to_dict(self):
+        generator = LlamaCppChatGenerator(model="test_model.gguf", n_ctx=8192, n_batch=512)
+        assert generator.to_dict() == {
+            "type": "haystack_integrations.components.generators.llama_cpp.chat.chat_generator.LlamaCppChatGenerator",
+            "init_parameters": {
+                "model": "test_model.gguf",
+                "n_ctx": 8192,
+                "n_batch": 512,
+                "model_kwargs": {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512},
+                "generation_kwargs": {},
+                "tools": None,
+            },
+        }
+
+    def test_to_dict_with_toolset(self, temperature_tool):
+        toolset = Toolset([temperature_tool])
+        generator = LlamaCppChatGenerator(model="test_model.gguf", tools=toolset)
+
+        data = generator.to_dict()
+
+        assert "tools" in data["init_parameters"]["tools"]["data"]
+        assert data["init_parameters"]["tools"]["type"] == "haystack.tools.toolset.Toolset"
+
+    def test_from_dict_with_toolset(self, temperature_tool):
+        toolset = Toolset([temperature_tool])
+        generator = LlamaCppChatGenerator(model="test_model.gguf", tools=toolset)
+        data = generator.to_dict()
+
+        deserialized_component = LlamaCppChatGenerator.from_dict(data)
+
+        assert isinstance(deserialized_component.tools, Toolset)
+        assert all(isinstance(tool, Tool) for tool in deserialized_component.tools)
+
+    def test_from_dict(self):
+        serialized = {
+            "type": "haystack_integrations.components.generators.llama_cpp.chat.chat_generator.LlamaCppChatGenerator",
+            "init_parameters": {
+                "model": "test_model.gguf",
+                "n_ctx": 8192,
+                "n_batch": 512,
+                "model_kwargs": {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512},
+                "generation_kwargs": {},
+                "tools": None,
+            },
+        }
+        deserialized = LlamaCppChatGenerator.from_dict(serialized)
+        assert deserialized.model_path == "test_model.gguf"
+        assert deserialized.n_ctx == 8192
+        assert deserialized.n_batch == 512
+        assert deserialized.model_kwargs == {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512}
+        assert deserialized.generation_kwargs == {}
+
     def test_ignores_model_path_if_specified_in_model_kwargs(self):
         """
         Test that model_path is ignored if already specified in model_kwargs.
@@ -320,17 +452,6 @@ class TestLlamaCppChatGenerator:
 
 
 class TestLlamaCppChatGeneratorFunctionary:
-    def get_current_temperature(self, location):
-        """Get the current temperature in a given location"""
-        if "tokyo" in location.lower():
-            return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
-        elif "san francisco" in location.lower():
-            return json.dumps({"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"})
-        elif "paris" in location.lower():
-            return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
-        else:
-            return json.dumps({"location": location, "temperature": "unknown"})
-
     @pytest.fixture
     def generator(self, model_path, capsys):
         gguf_model_path = (
@@ -354,90 +475,60 @@ class TestLlamaCppChatGeneratorFunctionary:
 
     @pytest.mark.integration
     def test_function_call(self, generator):
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "get_user_info",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "username": {"type": "string", "description": "The username to retrieve information for."}
-                        },
-                        "required": ["username"],
-                    },
-                    "description": "Retrieves detailed information about a user.",
-                },
-            }
-        ]
+        def get_user_info(username: Annotated[str, "The username to retrieve information for."]):
+            """Retrieves detailed information about a user."""
+            return {"username": username, "age": 25, "location": "San Francisco"}
+
+        tool = create_tool_from_function(get_user_info)
+
         tool_choice = {"type": "function", "function": {"name": "get_user_info"}}
 
         messages = [
             ChatMessage.from_user("Get information for user john_doe"),
         ]
-        response = generator.run(messages=messages, generation_kwargs={"tools": tools, "tool_choice": tool_choice})
+        response = generator.run(messages=messages, tools=[tool], generation_kwargs={"tool_choice": tool_choice})
+
+        reply = response["replies"][0]
 
-        assert "tool_calls" in response["replies"][0].meta
-        tool_calls = response["replies"][0].meta["tool_calls"]
+        assert reply.role == ChatRole.ASSISTANT
+        assert reply.tool_calls
+        tool_calls = reply.tool_calls
         assert len(tool_calls) > 0
-        assert tool_calls[0]["function"]["name"] == "get_user_info"
-        assert "username" in json.loads(tool_calls[0]["function"]["arguments"])
-        assert response["replies"][0].role == ChatRole.ASSISTANT
+        assert tool_calls[0].tool_name == "get_user_info"
+        assert tool_calls[0].arguments == {"username": "john_doe"}
 
-    def test_function_call_and_execute(self, generator):
-        messages = [ChatMessage.from_user("What's the weather like in San Francisco?")]
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "get_current_temperature",
-                    "description": "Get the current temperature in a given location",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "location": {
-                                "type": "string",
-                                "description": "The city and state, e.g. San Francisco, CA",
-                            },
-                        },
-                        "required": ["location"],
-                    },
-                },
-            }
-        ]
+    @pytest.mark.integration
+    def test_function_call_and_execute(self, generator, temperature_tool):
+        user_message = ChatMessage.from_user("What's the weather like in San Francisco?")
 
         tool_choice = {"type": "function", "function": {"name": "get_current_temperature"}}
-        response = generator.run(messages=messages, generation_kwargs={"tools": tools, "tool_choice": tool_choice})
-
-        available_functions = {
-            "get_current_temperature": self.get_current_temperature,
-        }
+        response = generator.run(
+            messages=[user_message], tools=[temperature_tool], generation_kwargs={"tool_choice": tool_choice}
+        )
 
         assert "replies" in response
         assert len(response["replies"]) > 0
-
         first_reply = response["replies"][0]
-        assert "tool_calls" in first_reply.meta
-        tool_calls = first_reply.meta["tool_calls"]
+        assert first_reply.tool_calls
+        tool_calls = first_reply.tool_calls
 
-        if hasattr(ChatMessage, "from_function"):
-            for tool_call in tool_calls:
-                function_name = tool_call["function"]["name"]
-                function_args = json.loads(tool_call["function"]["arguments"])
-                assert function_name in available_functions
-                function_response = available_functions[function_name](**function_args)
-                function_message = ChatMessage.from_function(function_response, function_name)
-                messages.append(function_message)
+        # tool invocation
+        tool_call = tool_calls[0]
+        function_args = tool_call.arguments
+        tool_response = str(temperature_tool.invoke(**function_args))
 
-            second_response = generator.run(messages=messages)
-            assert "replies" in second_response
-            assert len(second_response["replies"]) > 0
-            assert any("San Francisco" in reply.text for reply in second_response["replies"])
-            assert any("72" in reply.text for reply in second_response["replies"])
+        tool_message = ChatMessage.from_tool(tool_result=tool_response, origin=tool_call)
 
+        all_messages = [user_message, first_reply, tool_message]
+
+        second_response = generator.run(messages=all_messages)
+        assert "replies" in second_response
+        assert len(second_response["replies"]) > 0
+        assert any("San Francisco" in reply.text for reply in second_response["replies"])
+        assert any("72" in reply.text for reply in second_response["replies"])
 
-class TestLlamaCppChatGeneratorChatML:
 
+class TestLlamaCppChatGeneratorChatML:
     @pytest.fixture
     def generator(self, model_path, capsys):
         gguf_model_path = (
@@ -459,42 +550,32 @@ class TestLlamaCppChatGeneratorChatML:
 
     @pytest.mark.integration
     def test_function_call_chatml(self, generator):
+        def get_user_detail(name: Annotated[str, "The name of the user"], age: Annotated[int, "The age of the user"]):
+            """Retrieves detailed information about a user."""
+            pass
+
+        tool = create_tool_from_function(get_user_detail)
+
         messages = [
             ChatMessage.from_system(
                 """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful,
                 detailed, and polite answers to the user's questions. The assistant calls functions with appropriate
                 input when necessary"""
             ),
-            ChatMessage.from_user("Extract Jason is 25 years old"),
+            ChatMessage.from_user("Get details for user: Jason who is 25 years old"),
         ]
 
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "UserDetail",
-                    "parameters": {
-                        "type": "object",
-                        "title": "UserDetail",
-                        "properties": {
-                            "name": {"title": "Name", "type": "string"},
-                            "age": {"title": "Age", "type": "integer"},
-                        },
-                        "required": ["name", "age"],
-                    },
-                },
-            }
-        ]
+        tool_choice = {"type": "function", "function": {"name": "get_user_detail"}}
 
-        tool_choice = {"type": "function", "function": {"name": "UserDetail"}}
-
-        response = generator.run(messages=messages, generation_kwargs={"tools": tools, "tool_choice": tool_choice})
-        for reply in response["replies"]:
-            assert "tool_calls" in reply.meta
-            tool_calls = reply.meta["tool_calls"]
-            assert len(tool_calls) > 0
-            assert tool_calls[0]["function"]["name"] == "UserDetail"
-            assert "name" in json.loads(tool_calls[0]["function"]["arguments"])
-            assert "age" in json.loads(tool_calls[0]["function"]["arguments"])
-            assert "Jason" in json.loads(tool_calls[0]["function"]["arguments"])["name"]
-            assert 25 == json.loads(tool_calls[0]["function"]["arguments"])["age"]
+        response = generator.run(messages=messages, tools=[tool], generation_kwargs={"tool_choice": tool_choice})
+
+        reply = response["replies"][0]
+        assert reply.tool_calls
+        tool_calls = reply.tool_calls
+        assert len(tool_calls) > 0
+        assert tool_calls[0].tool_name == "get_user_detail"
+        arguments = tool_calls[0].arguments
+        assert "name" in arguments
+        assert "age" in arguments
+        assert arguments["name"] == "Jason"
+        assert arguments["age"] == 25

llama_cpp_haystack-0.4.4/src/haystack_integrations/components/generators/llama_cpp/chat/chat_generator.py

@@ -1,144 +0,0 @@
-import logging
-from typing import Any, Dict, List, Optional
-
-from haystack import component
-from haystack.dataclasses import ChatMessage
-from llama_cpp import Llama
-from llama_cpp.llama_tokenizer import LlamaHFTokenizer
-
-logger = logging.getLogger(__name__)
-
-
-def _convert_message_to_llamacpp_format(message: ChatMessage) -> Dict[str, str]:
-    """
-    Convert a message to the format expected by Llama.cpp.
-    :returns: A dictionary with the following keys:
-        - `role`
-        - `content`
-        - `name` (optional)
-    """
-    formatted_msg = {"role": message.role.value, "content": message.text}
-    if message.name:
-        formatted_msg["name"] = message.name
-
-    if formatted_msg["role"] == "tool":
-        formatted_msg["name"] = message.tool_call_result.origin.tool_name
-        formatted_msg["content"] = message.tool_call_result.result
-
-    return formatted_msg
-
-
-@component
-class LlamaCppChatGenerator:
-    """
-    Provides an interface to generate text using LLM via llama.cpp.
-
-    [llama.cpp](https://github.com/ggerganov/llama.cpp) is a project written in C/C++ for efficient inference of LLMs.
-    It employs the quantized GGUF format, suitable for running these models on standard machines (even without GPUs).
-
-    Usage example:
-    ```python
-    from haystack_integrations.components.generators.llama_cpp import LlamaCppChatGenerator
-    user_message = [ChatMessage.from_user("Who is the best American actor?")]
-    generator = LlamaCppGenerator(model="zephyr-7b-beta.Q4_0.gguf", n_ctx=2048, n_batch=512)
-
-    print(generator.run(user_message, generation_kwargs={"max_tokens": 128}))
-    # {"replies": [ChatMessage(content="John Cusack", role=<ChatRole.ASSISTANT: "assistant">, name=None, meta={...}]}
-    ```
-    """
-
-    def __init__(
-        self,
-        model: str,
-        n_ctx: Optional[int] = 0,
-        n_batch: Optional[int] = 512,
-        model_kwargs: Optional[Dict[str, Any]] = None,
-        generation_kwargs: Optional[Dict[str, Any]] = None,
-    ):
-        """
-        :param model: The path of a quantized model for text generation, for example, "zephyr-7b-beta.Q4_0.gguf".
-            If the model path is also specified in the `model_kwargs`, this parameter will be ignored.
-        :param n_ctx: The number of tokens in the context. When set to 0, the context will be taken from the model.
-        :param n_batch: Prompt processing maximum batch size.
-        :param model_kwargs: Dictionary containing keyword arguments used to initialize the LLM for text generation.
-            These keyword arguments provide fine-grained control over the model loading.
-            In case of duplication, these kwargs override `model`, `n_ctx`, and `n_batch` init parameters.
-            For more information on the available kwargs, see
-            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.__init__).
-        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
-            For more information on the available kwargs, see
-            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
-        """
-
-        model_kwargs = model_kwargs or {}
-        generation_kwargs = generation_kwargs or {}
-
-        if "hf_tokenizer_path" in model_kwargs:
-            tokenizer = LlamaHFTokenizer.from_pretrained(model_kwargs["hf_tokenizer_path"])
-            model_kwargs["tokenizer"] = tokenizer
-
-        # check if the model_kwargs contain the essential parameters
-        # otherwise, populate them with values from init parameters
-        model_kwargs.setdefault("model_path", model)
-        model_kwargs.setdefault("n_ctx", n_ctx)
-        model_kwargs.setdefault("n_batch", n_batch)
-
-        self.model_path = model
-        self.n_ctx = n_ctx
-        self.n_batch = n_batch
-        self.model_kwargs = model_kwargs
-        self.generation_kwargs = generation_kwargs
-        self.model = None
-
-    def warm_up(self):
-        if self.model is None:
-            self.model = Llama(**self.model_kwargs)
-
-    @component.output_types(replies=List[ChatMessage])
-    def run(self, messages: List[ChatMessage], generation_kwargs: Optional[Dict[str, Any]] = None):
-        """
-        Run the text generation model on the given list of ChatMessages.
-
-        :param messages:
-            A list of ChatMessage instances representing the input messages.
-        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
-            For more information on the available kwargs, see
-            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
-        :returns: A dictionary with the following keys:
-            - `replies`: The responses from the model
-        """
-        if self.model is None:
-            error_msg = "The model has not been loaded. Please call warm_up() before running."
-            raise RuntimeError(error_msg)
-
-        if not messages:
-            return {"replies": []}
-
-        updated_generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
-        formatted_messages = [_convert_message_to_llamacpp_format(msg) for msg in messages]
-
-        response = self.model.create_chat_completion(messages=formatted_messages, **updated_generation_kwargs)
-
-        replies = []
-
-        for choice in response["choices"]:
-            meta = {
-                "response_id": response["id"],
-                "model": response["model"],
-                "created": response["created"],
-                "index": choice["index"],
-                "finish_reason": choice["finish_reason"],
-                "usage": response["usage"],
-            }
-
-            name = None
-            tool_calls = choice.get("message", {}).get("tool_calls", [])
-            if tool_calls:
-                meta["tool_calls"] = tool_calls
-                name = tool_calls[0]["function"]["name"]
-
-            reply = ChatMessage.from_assistant(choice["message"]["content"], meta=meta)
-            reply._name = name or None
-            replies.append(reply)
-
-        return {"replies": replies}