llama-cpp-haystack 0.4.4__tar.gz → 1.0.0__tar.gz

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.0.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [integrations/llama_cpp-v0.4.4] - 2025-01-16
+
+### 🧹 Chores
+
+- Llama.cpp - gently handle the removal of ChatMessage.from_function (#1298)
+
+
 ## [integrations/llama_cpp-v0.4.3] - 2024-12-19
 
 ### 🐛 Bug Fixes

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llama-cpp-haystack
-Version: 0.4.4
+Version: 1.0.0
 Summary: An integration between the llama.cpp LLM framework and Haystack
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/llama_cpp#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.0.0}/pyproject.toml

@@ -52,6 +52,7 @@ dependencies = [
     "pytest-rerunfailures",
     "haystack-pydoc-tools",
     "transformers[sentencepiece]",
+    "jsonschema",  # needed for Tool
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"

llama_cpp_haystack-1.0.0/src/haystack_integrations/components/generators/llama_cpp/chat/chat_generator.py

@@ -0,0 +1,251 @@
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage, ToolCall
+from haystack.tools import Tool, _check_duplicate_tool_names, deserialize_tools_inplace
+from llama_cpp import ChatCompletionResponseChoice, CreateChatCompletionResponse, Llama
+from llama_cpp.llama_tokenizer import LlamaHFTokenizer
+
+logger = logging.getLogger(__name__)
+
+
+def _convert_message_to_llamacpp_format(message: ChatMessage) -> Dict[str, Any]:
+    """
+    Convert a ChatMessage to the format expected by Ollama Chat API.
+    """
+    text_contents = message.texts
+    tool_calls = message.tool_calls
+    tool_call_results = message.tool_call_results
+
+    if not text_contents and not tool_calls and not tool_call_results:
+        msg = "A `ChatMessage` must contain at least one `TextContent`, `ToolCall`, or `ToolCallResult`."
+        raise ValueError(msg)
+    elif len(text_contents) + len(tool_call_results) > 1:
+        msg = "A `ChatMessage` can only contain one `TextContent` or one `ToolCallResult`."
+        raise ValueError(msg)
+
+    role = message._role.value
+    if role == "tool":
+        role = "function"
+
+    llamacpp_msg: Dict[str, Any] = {"role": role}
+
+    if tool_call_results:
+        if tool_call_results[0].origin.id is None:
+            msg = "`ToolCall` must have a non-null `id` attribute to be used with llama.cpp."
+            raise ValueError(msg)
+        llamacpp_msg["content"] = tool_call_results[0].result
+        llamacpp_msg["tool_call_id"] = tool_call_results[0].origin.id
+        # Llama.cpp does not provide a way to communicate errors in tool invocations, so we ignore the error field
+        return llamacpp_msg
+
+    if text_contents:
+        llamacpp_msg["content"] = text_contents[0]
+    if tool_calls:
+        llamacpp_tool_calls = []
+        for tc in tool_calls:
+            if tc.id is None:
+                msg = "`ToolCall` must have a non-null `id` attribute to be used with llama.cpp."
+                raise ValueError(msg)
+            llamacpp_tool_calls.append(
+                {
+                    "id": tc.id,
+                    "type": "function",
+                    # We disable ensure_ascii so special chars like emojis are not converted
+                    "function": {"name": tc.tool_name, "arguments": json.dumps(tc.arguments, ensure_ascii=False)},
+                }
+            )
+        llamacpp_msg["tool_calls"] = llamacpp_tool_calls
+    return llamacpp_msg
+
+
+@component
+class LlamaCppChatGenerator:
+    """
+    Provides an interface to generate text using LLM via llama.cpp.
+
+    [llama.cpp](https://github.com/ggerganov/llama.cpp) is a project written in C/C++ for efficient inference of LLMs.
+    It employs the quantized GGUF format, suitable for running these models on standard machines (even without GPUs).
+
+    Usage example:
+    ```python
+    from haystack_integrations.components.generators.llama_cpp import LlamaCppChatGenerator
+    user_message = [ChatMessage.from_user("Who is the best American actor?")]
+    generator = LlamaCppGenerator(model="zephyr-7b-beta.Q4_0.gguf", n_ctx=2048, n_batch=512)
+
+    print(generator.run(user_message, generation_kwargs={"max_tokens": 128}))
+    # {"replies": [ChatMessage(content="John Cusack", role=<ChatRole.ASSISTANT: "assistant">, name=None, meta={...}]}
+    ```
+    """
+
+    def __init__(
+        self,
+        model: str,
+        n_ctx: Optional[int] = 0,
+        n_batch: Optional[int] = 512,
+        model_kwargs: Optional[Dict[str, Any]] = None,
+        generation_kwargs: Optional[Dict[str, Any]] = None,
+        *,
+        tools: Optional[List[Tool]] = None,
+    ):
+        """
+        :param model: The path of a quantized model for text generation, for example, "zephyr-7b-beta.Q4_0.gguf".
+            If the model path is also specified in the `model_kwargs`, this parameter will be ignored.
+        :param n_ctx: The number of tokens in the context. When set to 0, the context will be taken from the model.
+        :param n_batch: Prompt processing maximum batch size.
+        :param model_kwargs: Dictionary containing keyword arguments used to initialize the LLM for text generation.
+            These keyword arguments provide fine-grained control over the model loading.
+            In case of duplication, these kwargs override `model`, `n_ctx`, and `n_batch` init parameters.
+            For more information on the available kwargs, see
+            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.__init__).
+        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
+            For more information on the available kwargs, see
+            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
+        :param tools:
+            A list of tools for which the model can prepare calls.
+        """
+
+        model_kwargs = model_kwargs or {}
+        generation_kwargs = generation_kwargs or {}
+
+        # check if the model_kwargs contain the essential parameters
+        # otherwise, populate them with values from init parameters
+        model_kwargs.setdefault("model_path", model)
+        model_kwargs.setdefault("n_ctx", n_ctx)
+        model_kwargs.setdefault("n_batch", n_batch)
+
+        _check_duplicate_tool_names(tools)
+
+        self.model_path = model
+        self.n_ctx = n_ctx
+        self.n_batch = n_batch
+        self.model_kwargs = model_kwargs
+        self.generation_kwargs = generation_kwargs
+        self._model = None
+        self.tools = tools
+
+    def warm_up(self):
+        if "hf_tokenizer_path" in self.model_kwargs and "tokenizer" not in self.model_kwargs:
+            tokenizer = LlamaHFTokenizer.from_pretrained(self.model_kwargs["hf_tokenizer_path"])
+            self.model_kwargs["tokenizer"] = tokenizer
+
+        if self._model is None:
+            self._model = Llama(**self.model_kwargs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+              Dictionary with serialized data.
+        """
+        serialized_tools = [tool.to_dict() for tool in self.tools] if self.tools else None
+        return default_to_dict(
+            self,
+            model=self.model_path,
+            n_ctx=self.n_ctx,
+            n_batch=self.n_batch,
+            model_kwargs=self.model_kwargs,
+            generation_kwargs=self.generation_kwargs,
+            tools=serialized_tools,
+        )
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "LlamaCppChatGenerator":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        deserialize_tools_inplace(data["init_parameters"], key="tools")
+        return default_from_dict(cls, data)
+
+    @component.output_types(replies=List[ChatMessage])
+    def run(
+        self,
+        messages: List[ChatMessage],
+        generation_kwargs: Optional[Dict[str, Any]] = None,
+        *,
+        tools: Optional[List[Tool]] = None,
+    ):
+        """
+        Run the text generation model on the given list of ChatMessages.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages.
+        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
+            For more information on the available kwargs, see
+            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
+        :param tools:
+            A list of tools for which the model can prepare calls. If set, it will override the `tools` parameter set
+            during component initialization.
+        :returns: A dictionary with the following keys:
+            - `replies`: The responses from the model
+        """
+        if self._model is None:
+            error_msg = "The model has not been loaded. Please call warm_up() before running."
+            raise RuntimeError(error_msg)
+
+        if not messages:
+            return {"replies": []}
+
+        updated_generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
+        formatted_messages = [_convert_message_to_llamacpp_format(msg) for msg in messages]
+
+        tools = tools or self.tools
+        llamacpp_tools = {}
+        if tools:
+            tool_definitions = [{"type": "function", "function": {**t.tool_spec}} for t in tools]
+            llamacpp_tools = {"tools": tool_definitions}
+
+        response = self._model.create_chat_completion(
+            messages=formatted_messages, **updated_generation_kwargs, **llamacpp_tools
+        )
+
+        replies = []
+        for choice in response["choices"]:
+            chat_message = self._convert_chat_completion_choice_to_chat_message(choice, response)
+            replies.append(chat_message)
+
+        return {"replies": replies}
+
+    @staticmethod
+    def _convert_chat_completion_choice_to_chat_message(
+        choice: ChatCompletionResponseChoice, response: CreateChatCompletionResponse
+    ) -> ChatMessage:
+        llamacpp_message = choice["message"]
+        text_content = llamacpp_message.get("content", "") or None
+        tool_calls = []
+
+        if llamacpp_tool_calls := llamacpp_message.get("tool_calls", []):
+            for llamacpp_tc in llamacpp_tool_calls:
+                arguments_str = llamacpp_tc["function"]["arguments"]
+                try:
+                    arguments = json.loads(arguments_str)
+                    tool_calls.append(
+                        ToolCall(id=llamacpp_tc["id"], tool_name=llamacpp_tc["function"]["name"], arguments=arguments)
+                    )
+                except json.JSONDecodeError:
+                    logger.warning(
+                        "Llama.cpp returned a malformed JSON string for tool call arguments. This tool call "
+                        "will be skipped. Tool call ID: %s, Tool name: %s, Arguments: %s",
+                        llamacpp_tc["id"],
+                        llamacpp_tc["function"]["name"],
+                        arguments_str,
+                    )
+
+        meta = {
+            "response_id": response["id"],
+            "model": response["model"],
+            "created": response["created"],
+            "index": choice["index"],
+            "finish_reason": choice["finish_reason"],
+            "usage": response["usage"],
+        }
+
+        return ChatMessage.from_assistant(text=text_content, tool_calls=tool_calls, meta=meta)

{llama_cpp_haystack-0.4.4 → llama_cpp_haystack-1.0.0}/tests/test_chat_generator.py

@@ -2,14 +2,16 @@ import json
 import os
 import urllib.request
 from pathlib import Path
+from typing import Annotated
 from unittest.mock import MagicMock
 
 import pytest
 from haystack import Document, Pipeline
 from haystack.components.builders import ChatPromptBuilder
 from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
-from haystack.dataclasses import ChatMessage, ChatRole
+from haystack.dataclasses import ChatMessage, ChatRole, TextContent, ToolCall
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.tools import create_tool_from_function
 
 from haystack_integrations.components.generators.llama_cpp.chat.chat_generator import (
     LlamaCppChatGenerator,
@@ -35,17 +37,72 @@ def download_file(file_link, filename, capsys):
 
 def test_convert_message_to_llamacpp_format():
     message = ChatMessage.from_system("You are good assistant")
-    assert _convert_message_to_llamacpp_format(message) == {"role": "system", "content": "You are good assistant"}
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "system",
+        "content": "You are good assistant",
+    }
 
     message = ChatMessage.from_user("I have a question")
-    assert _convert_message_to_llamacpp_format(message) == {"role": "user", "content": "I have a question"}
-
-    if hasattr(ChatMessage, "from_function"):
-        message = ChatMessage.from_function("Function call", "function_name")
-        converted_message = _convert_message_to_llamacpp_format(message)
-        assert converted_message["role"] in ("function", "tool")
-        assert converted_message["name"] == "function_name"
-        assert converted_message["content"] == "Function call"
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "user",
+        "content": "I have a question",
+    }
+
+    message = ChatMessage.from_assistant(text="I have an answer", meta={"finish_reason": "stop"})
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "assistant",
+        "content": "I have an answer",
+    }
+
+    message = ChatMessage.from_assistant(
+        tool_calls=[ToolCall(id="123", tool_name="weather", arguments={"city": "Paris"})]
+    )
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "assistant",
+        "tool_calls": [
+            {
+                "type": "function",
+                "function": {"name": "weather", "arguments": '{"city": "Paris"}'},
+                "id": "123",
+            }
+        ],
+    }
+
+    tool_result = json.dumps({"weather": "sunny", "temperature": "25"})
+    message = ChatMessage.from_tool(
+        tool_result=tool_result,
+        origin=ToolCall(id="123", tool_name="weather", arguments={"city": "Paris"}),
+    )
+    assert _convert_message_to_llamacpp_format(message) == {
+        "role": "function",
+        "content": tool_result,
+        "tool_call_id": "123",
+    }
+
+
+def test_convert_message_to_llamacpp_invalid():
+    message = ChatMessage(_role=ChatRole.ASSISTANT, _content=[])
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
+
+    message = ChatMessage(
+        _role=ChatRole.ASSISTANT,
+        _content=[
+            TextContent(text="I have an answer"),
+            TextContent(text="I have another answer"),
+        ],
+    )
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
+
+    tool_call_null_id = ToolCall(id=None, tool_name="weather", arguments={"city": "Paris"})
+    message = ChatMessage.from_assistant(tool_calls=[tool_call_null_id])
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
+
+    message = ChatMessage.from_tool(tool_result="result", origin=tool_call_null_id)
+    with pytest.raises(ValueError):
+        _convert_message_to_llamacpp_format(message)
 
 
 class TestLlamaCppChatGenerator:
@@ -68,7 +125,7 @@ class TestLlamaCppChatGenerator:
     def generator_mock(self):
         mock_model = MagicMock()
         generator = LlamaCppChatGenerator(model="test_model.gguf", n_ctx=2048, n_batch=512)
-        generator.model = mock_model
+        generator._model = mock_model
         return generator, mock_model
 
     def test_default_init(self):
@@ -99,6 +156,39 @@ class TestLlamaCppChatGenerator:
         assert generator.model_kwargs == {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512}
         assert generator.generation_kwargs == {}
 
+    def test_to_dict(self):
+        generator = LlamaCppChatGenerator(model="test_model.gguf", n_ctx=8192, n_batch=512)
+        assert generator.to_dict() == {
+            "type": "haystack_integrations.components.generators.llama_cpp.chat.chat_generator.LlamaCppChatGenerator",
+            "init_parameters": {
+                "model": "test_model.gguf",
+                "n_ctx": 8192,
+                "n_batch": 512,
+                "model_kwargs": {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512},
+                "generation_kwargs": {},
+                "tools": None,
+            },
+        }
+
+    def test_from_dict(self):
+        serialized = {
+            "type": "haystack_integrations.components.generators.llama_cpp.chat.chat_generator.LlamaCppChatGenerator",
+            "init_parameters": {
+                "model": "test_model.gguf",
+                "n_ctx": 8192,
+                "n_batch": 512,
+                "model_kwargs": {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512},
+                "generation_kwargs": {},
+                "tools": None,
+            },
+        }
+        deserialized = LlamaCppChatGenerator.from_dict(serialized)
+        assert deserialized.model_path == "test_model.gguf"
+        assert deserialized.n_ctx == 8192
+        assert deserialized.n_batch == 512
+        assert deserialized.model_kwargs == {"model_path": "test_model.gguf", "n_ctx": 8192, "n_batch": 512}
+        assert deserialized.generation_kwargs == {}
+
     def test_ignores_model_path_if_specified_in_model_kwargs(self):
         """
         Test that model_path is ignored if already specified in model_kwargs.
@@ -320,16 +410,17 @@ class TestLlamaCppChatGenerator:
 
 
 class TestLlamaCppChatGeneratorFunctionary:
-    def get_current_temperature(self, location):
+    def get_current_temperature(self, location: Annotated[str, "The city and state, e.g. San Francisco, CA"]):
         """Get the current temperature in a given location"""
+
         if "tokyo" in location.lower():
-            return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
-        elif "san francisco" in location.lower():
-            return json.dumps({"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"})
-        elif "paris" in location.lower():
-            return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
-        else:
-            return json.dumps({"location": location, "temperature": "unknown"})
+            return {"location": "Tokyo", "temperature": "10", "unit": "celsius"}
+        if "san francisco" in location.lower():
+            return {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}
+        if "paris" in location.lower():
+            return {"location": "Paris", "temperature": "22", "unit": "celsius"}
+
+        return {"location": location, "temperature": "unknown"}
 
     @pytest.fixture
     def generator(self, model_path, capsys):
@@ -354,86 +445,60 @@ class TestLlamaCppChatGeneratorFunctionary:
 
     @pytest.mark.integration
     def test_function_call(self, generator):
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "get_user_info",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "username": {"type": "string", "description": "The username to retrieve information for."}
-                        },
-                        "required": ["username"],
-                    },
-                    "description": "Retrieves detailed information about a user.",
-                },
-            }
-        ]
+
+        def get_user_info(username: Annotated[str, "The username to retrieve information for."]):
+            """Retrieves detailed information about a user."""
+            return {"username": username, "age": 25, "location": "San Francisco"}
+
+        tool = create_tool_from_function(get_user_info)
+
         tool_choice = {"type": "function", "function": {"name": "get_user_info"}}
 
         messages = [
             ChatMessage.from_user("Get information for user john_doe"),
         ]
-        response = generator.run(messages=messages, generation_kwargs={"tools": tools, "tool_choice": tool_choice})
+        response = generator.run(messages=messages, tools=[tool], generation_kwargs={"tool_choice": tool_choice})
+
+        reply = response["replies"][0]
 
-        assert "tool_calls" in response["replies"][0].meta
-        tool_calls = response["replies"][0].meta["tool_calls"]
+        assert reply.role == ChatRole.ASSISTANT
+        assert reply.tool_calls
+        tool_calls = reply.tool_calls
         assert len(tool_calls) > 0
-        assert tool_calls[0]["function"]["name"] == "get_user_info"
-        assert "username" in json.loads(tool_calls[0]["function"]["arguments"])
-        assert response["replies"][0].role == ChatRole.ASSISTANT
+        assert tool_calls[0].tool_name == "get_user_info"
+        assert tool_calls[0].arguments == {"username": "john_doe"}
 
     def test_function_call_and_execute(self, generator):
-        messages = [ChatMessage.from_user("What's the weather like in San Francisco?")]
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "get_current_temperature",
-                    "description": "Get the current temperature in a given location",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "location": {
-                                "type": "string",
-                                "description": "The city and state, e.g. San Francisco, CA",
-                            },
-                        },
-                        "required": ["location"],
-                    },
-                },
-            }
-        ]
+        temperature_tool = create_tool_from_function(self.get_current_temperature)
 
-        tool_choice = {"type": "function", "function": {"name": "get_current_temperature"}}
-        response = generator.run(messages=messages, generation_kwargs={"tools": tools, "tool_choice": tool_choice})
+        user_message = ChatMessage.from_user("What's the weather like in San Francisco?")
 
-        available_functions = {
-            "get_current_temperature": self.get_current_temperature,
-        }
+        tool_choice = {"type": "function", "function": {"name": "get_current_temperature"}}
+        response = generator.run(
+            messages=[user_message], tools=[temperature_tool], generation_kwargs={"tool_choice": tool_choice}
+        )
 
         assert "replies" in response
         assert len(response["replies"]) > 0
-
         first_reply = response["replies"][0]
-        assert "tool_calls" in first_reply.meta
-        tool_calls = first_reply.meta["tool_calls"]
+        assert first_reply.tool_calls
+        tool_calls = first_reply.tool_calls
 
-        if hasattr(ChatMessage, "from_function"):
-            for tool_call in tool_calls:
-                function_name = tool_call["function"]["name"]
-                function_args = json.loads(tool_call["function"]["arguments"])
-                assert function_name in available_functions
-                function_response = available_functions[function_name](**function_args)
-                function_message = ChatMessage.from_function(function_response, function_name)
-                messages.append(function_message)
+        # tool invocation
+        tool_call = tool_calls[0]
+        function_args = tool_call.arguments
+        tool_response = str(temperature_tool.invoke(**function_args))
 
-            second_response = generator.run(messages=messages)
-            assert "replies" in second_response
-            assert len(second_response["replies"]) > 0
-            assert any("San Francisco" in reply.text for reply in second_response["replies"])
-            assert any("72" in reply.text for reply in second_response["replies"])
+        tool_message = ChatMessage.from_tool(tool_result=tool_response, origin=tool_call)
+
+        all_messages = [user_message, first_reply, tool_message]
+        print(all_messages)
+
+        second_response = generator.run(messages=all_messages)
+        assert "replies" in second_response
+        assert len(second_response["replies"]) > 0
+        assert any("San Francisco" in reply.text for reply in second_response["replies"])
+        assert any("72" in reply.text for reply in second_response["replies"])
 
 
 class TestLlamaCppChatGeneratorChatML:
@@ -459,42 +524,33 @@ class TestLlamaCppChatGeneratorChatML:
 
     @pytest.mark.integration
     def test_function_call_chatml(self, generator):
+
+        def get_user_detail(name: Annotated[str, "The name of the user"], age: Annotated[int, "The age of the user"]):
+            """Retrieves detailed information about a user."""
+            pass
+
+        tool = create_tool_from_function(get_user_detail)
+
         messages = [
             ChatMessage.from_system(
                 """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful,
                 detailed, and polite answers to the user's questions. The assistant calls functions with appropriate
                 input when necessary"""
             ),
-            ChatMessage.from_user("Extract Jason is 25 years old"),
+            ChatMessage.from_user("Get details for user: Jason who is 25 years old"),
         ]
 
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "UserDetail",
-                    "parameters": {
-                        "type": "object",
-                        "title": "UserDetail",
-                        "properties": {
-                            "name": {"title": "Name", "type": "string"},
-                            "age": {"title": "Age", "type": "integer"},
-                        },
-                        "required": ["name", "age"],
-                    },
-                },
-            }
-        ]
+        tool_choice = {"type": "function", "function": {"name": "get_user_detail"}}
+
+        response = generator.run(messages=messages, tools=[tool], generation_kwargs={"tool_choice": tool_choice})
 
-        tool_choice = {"type": "function", "function": {"name": "UserDetail"}}
-
-        response = generator.run(messages=messages, generation_kwargs={"tools": tools, "tool_choice": tool_choice})
-        for reply in response["replies"]:
-            assert "tool_calls" in reply.meta
-            tool_calls = reply.meta["tool_calls"]
-            assert len(tool_calls) > 0
-            assert tool_calls[0]["function"]["name"] == "UserDetail"
-            assert "name" in json.loads(tool_calls[0]["function"]["arguments"])
-            assert "age" in json.loads(tool_calls[0]["function"]["arguments"])
-            assert "Jason" in json.loads(tool_calls[0]["function"]["arguments"])["name"]
-            assert 25 == json.loads(tool_calls[0]["function"]["arguments"])["age"]
+        reply = response["replies"][0]
+        assert reply.tool_calls
+        tool_calls = reply.tool_calls
+        assert len(tool_calls) > 0
+        assert tool_calls[0].tool_name == "get_user_detail"
+        arguments = tool_calls[0].arguments
+        assert "name" in arguments
+        assert "age" in arguments
+        assert arguments["name"] == "Jason"
+        assert arguments["age"] == 25

llama_cpp_haystack-0.4.4/src/haystack_integrations/components/generators/llama_cpp/chat/chat_generator.py

@@ -1,144 +0,0 @@
-import logging
-from typing import Any, Dict, List, Optional
-
-from haystack import component
-from haystack.dataclasses import ChatMessage
-from llama_cpp import Llama
-from llama_cpp.llama_tokenizer import LlamaHFTokenizer
-
-logger = logging.getLogger(__name__)
-
-
-def _convert_message_to_llamacpp_format(message: ChatMessage) -> Dict[str, str]:
-    """
-    Convert a message to the format expected by Llama.cpp.
-    :returns: A dictionary with the following keys:
-        - `role`
-        - `content`
-        - `name` (optional)
-    """
-    formatted_msg = {"role": message.role.value, "content": message.text}
-    if message.name:
-        formatted_msg["name"] = message.name
-
-    if formatted_msg["role"] == "tool":
-        formatted_msg["name"] = message.tool_call_result.origin.tool_name
-        formatted_msg["content"] = message.tool_call_result.result
-
-    return formatted_msg
-
-
-@component
-class LlamaCppChatGenerator:
-    """
-    Provides an interface to generate text using LLM via llama.cpp.
-
-    [llama.cpp](https://github.com/ggerganov/llama.cpp) is a project written in C/C++ for efficient inference of LLMs.
-    It employs the quantized GGUF format, suitable for running these models on standard machines (even without GPUs).
-
-    Usage example:
-    ```python
-    from haystack_integrations.components.generators.llama_cpp import LlamaCppChatGenerator
-    user_message = [ChatMessage.from_user("Who is the best American actor?")]
-    generator = LlamaCppGenerator(model="zephyr-7b-beta.Q4_0.gguf", n_ctx=2048, n_batch=512)
-
-    print(generator.run(user_message, generation_kwargs={"max_tokens": 128}))
-    # {"replies": [ChatMessage(content="John Cusack", role=<ChatRole.ASSISTANT: "assistant">, name=None, meta={...}]}
-    ```
-    """
-
-    def __init__(
-        self,
-        model: str,
-        n_ctx: Optional[int] = 0,
-        n_batch: Optional[int] = 512,
-        model_kwargs: Optional[Dict[str, Any]] = None,
-        generation_kwargs: Optional[Dict[str, Any]] = None,
-    ):
-        """
-        :param model: The path of a quantized model for text generation, for example, "zephyr-7b-beta.Q4_0.gguf".
-            If the model path is also specified in the `model_kwargs`, this parameter will be ignored.
-        :param n_ctx: The number of tokens in the context. When set to 0, the context will be taken from the model.
-        :param n_batch: Prompt processing maximum batch size.
-        :param model_kwargs: Dictionary containing keyword arguments used to initialize the LLM for text generation.
-            These keyword arguments provide fine-grained control over the model loading.
-            In case of duplication, these kwargs override `model`, `n_ctx`, and `n_batch` init parameters.
-            For more information on the available kwargs, see
-            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.__init__).
-        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
-            For more information on the available kwargs, see
-            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
-        """
-
-        model_kwargs = model_kwargs or {}
-        generation_kwargs = generation_kwargs or {}
-
-        if "hf_tokenizer_path" in model_kwargs:
-            tokenizer = LlamaHFTokenizer.from_pretrained(model_kwargs["hf_tokenizer_path"])
-            model_kwargs["tokenizer"] = tokenizer
-
-        # check if the model_kwargs contain the essential parameters
-        # otherwise, populate them with values from init parameters
-        model_kwargs.setdefault("model_path", model)
-        model_kwargs.setdefault("n_ctx", n_ctx)
-        model_kwargs.setdefault("n_batch", n_batch)
-
-        self.model_path = model
-        self.n_ctx = n_ctx
-        self.n_batch = n_batch
-        self.model_kwargs = model_kwargs
-        self.generation_kwargs = generation_kwargs
-        self.model = None
-
-    def warm_up(self):
-        if self.model is None:
-            self.model = Llama(**self.model_kwargs)
-
-    @component.output_types(replies=List[ChatMessage])
-    def run(self, messages: List[ChatMessage], generation_kwargs: Optional[Dict[str, Any]] = None):
-        """
-        Run the text generation model on the given list of ChatMessages.
-
-        :param messages:
-            A list of ChatMessage instances representing the input messages.
-        :param generation_kwargs:  A dictionary containing keyword arguments to customize text generation.
-            For more information on the available kwargs, see
-            [llama.cpp documentation](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion).
-        :returns: A dictionary with the following keys:
-            - `replies`: The responses from the model
-        """
-        if self.model is None:
-            error_msg = "The model has not been loaded. Please call warm_up() before running."
-            raise RuntimeError(error_msg)
-
-        if not messages:
-            return {"replies": []}
-
-        updated_generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
-        formatted_messages = [_convert_message_to_llamacpp_format(msg) for msg in messages]
-
-        response = self.model.create_chat_completion(messages=formatted_messages, **updated_generation_kwargs)
-
-        replies = []
-
-        for choice in response["choices"]:
-            meta = {
-                "response_id": response["id"],
-                "model": response["model"],
-                "created": response["created"],
-                "index": choice["index"],
-                "finish_reason": choice["finish_reason"],
-                "usage": response["usage"],
-            }
-
-            name = None
-            tool_calls = choice.get("message", {}).get("tool_calls", [])
-            if tool_calls:
-                meta["tool_calls"] = tool_calls
-                name = tool_calls[0]["function"]["name"]
-
-            reply = ChatMessage.from_assistant(choice["message"]["content"], meta=meta)
-            reply._name = name or None
-            replies.append(reply)
-
-        return {"replies": replies}