llama-cpp-python 0.2.49__tar.gz → 0.2.51__tar.gz

llama_cpp_python-0.2.51/.git/FETCH_HEAD

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5		'5fc4c1efb61883087e25c06100e258ca2e7d15f5' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.51/.git/HEAD

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hKNVNmajM3UlFQNmZZWUdpY3hYeTgyMVhWTHBSYzJwUVRQUw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hnTUJRc3BOZnpreWs3cnlrdHBtbjQ1MDUxS0dEQzFJeUU0ZQ==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.51/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 5fc4c1efb61883087e25c06100e258ca2e7d15f5 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913844 +0000	checkout: moving from master to refs/tags/v0.2.51

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+c39373398803c669056304090050fe3f44b41bf9

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hKNVNmajM3UlFQNmZZWUdpY3hYeTgyMVhWTHBSYzJwUVRQUw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hnTUJRc3BOZnpreWs3cnlrdHBtbjQ1MDUxS0dEQzFJeUU0ZQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+c39373398803c669056304090050fe3f44b41bf9 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	checkout: moving from master to c39373398803c669056304090050fe3f44b41bf9

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hKNVNmajM3UlFQNmZZWUdpY3hYeTgyMVhWTHBSYzJwUVRQUw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hnTUJRc3BOZnpreWs3cnlrdHBtbjQ1MDUxS0dEQzFJeUU0ZQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913846 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+c39373398803c669056304090050fe3f44b41bf9 refs/remotes/origin/master

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+c39373398803c669056304090050fe3f44b41bf9

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1 @@
+c39373398803c669056304090050fe3f44b41bf9

llama_cpp_python-0.2.51/.git/refs/tags/v0.2.51

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5

llama_cpp_python-0.2.51/.git/shallow

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/CHANGELOG.md

@@ -7,12 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.51]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@c39373398803c669056304090050fe3f44b41bf9
+- fix: Restore type hints for low-level api by @abetlen in 19234aa0dbd0c3c87656e65dd2b064665371925b
+
+## [0.2.50]
+
+- docs: Update Functionary OpenAI Server Readme by @jeffrey-fong in #1193
+- fix: LlamaHFTokenizer now receives pre_tokens by @abetlen in 47bad30dd716443652275099fa3851811168ff4a
+
 ## [0.2.49]
 
 - fix: module 'llama_cpp.llama_cpp' has no attribute 'c_uint8' in Llama.save_state by @abetlen in db776a885cd4c20811f22f8bd1a27ecc71dba927
 - feat: Auto detect Mixtral's slightly different format by @lukestanley in #1214
 
-
 ## [0.2.48]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@15499eb94227401bdc8875da6eb85c15d37068f7
@@ -151,7 +160,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - feat: Update llama.cpp to ggerganov/llama.cpp@b3a7c20b5c035250257d2b62851c379b159c899a
 - feat: Add `saiga` chat format by @femoiseev in #1050
 - feat: Added `chatglm3` chat format by @xaviviro in #1059
-- fix: Correct typo in README.md by @qeleb in (#1058) 
+- fix: Correct typo in README.md by @qeleb in (#1058)
 
 ## [0.2.26]
 
@@ -284,7 +293,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.2.11]
 
-- Fix bug in `llama_model_params` object has no attribute `logits_all` by @abetlen in d696251fbe40015e8616ea7a7d7ad5257fd1b896 
+- Fix bug in `llama_model_params` object has no attribute `logits_all` by @abetlen in d696251fbe40015e8616ea7a7d7ad5257fd1b896
 
 ## [0.2.10]
 
@@ -472,7 +481,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.1.60]
 
-NOTE: This release was deleted due to a bug  with the packaging system that caused pip installations to fail.
+NOTE: This release was deleted due to a bug with the packaging system that caused pip installations to fail.
 
 - Truncate max_tokens in create_completion so requested tokens doesn't exceed context size.
 - Temporarily disable cache for completion requests
@@ -496,4 +505,4 @@ NOTE: This release was deleted due to a bug  with the packaging system that caus
 - (misc) Added first version of the changelog
 - (server) Use async routes
 - (python-api) Use numpy for internal buffers to reduce memory usage and improve performance.
-- (python-api) Performance bug in stop sequence check slowing down streaming.
+- (python-api) Performance bug in stop sequence check slowing down streaming.

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.49
+Version: 0.2.51
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -408,14 +408,10 @@ To constrain the response further to a specific JSON Schema add the schema to th
 
 ### Function Calling
 
-The high-level API also provides a simple interface for function calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format.
-
-The gguf-converted files for functionary can be found here: [functionary-7b-v1](https://huggingface.co/abetlen/functionary-7b-v1-GGUF)
+The high-level API supports OpenAI compatible function and tool calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format.
 
 ```python
 >>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/functionary/llama-model.gguf", chat_format="functionary")
->>> # or 
 >>> llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling")
 >>> llm.create_chat_completion(
       messages = [
@@ -459,6 +455,25 @@ The gguf-converted files for functionary can be found here: [functionary-7b-v1](
 )
 ```
 
+<details>
+<summary>Functionary v2</summary>
+
+The various gguf-converted files for this set of models can be found [here](https://huggingface.co/meetkai). Functionary is able to intelligently call functions and also analyze any provided function outputs to generate coherent responses. All v2 models of functionary supports **parallel function calling**. You can provide either `functionary-v1` or `functionary-v2` for the `chat_format` when initializing the Llama class.
+
+Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is required to provide HF Tokenizer for functionary. The `LlamaHFTokenizer` class can be initialized and passed into the Llama class. This will override the default llama.cpp tokenizer used in Llama class. The tokenizer files are already included in the respective HF repositories hosting the gguf files.
+
+```python
+>>> from llama_cpp import Llama
+>>> from llama_cpp.llama_tokenizer import LlamaHFTokenizer
+>>> llm = Llama.from_pretrained(
+  repo_id="meetkai/functionary-small-v2.2-GGUF",
+  filename="functionary-small-v2.2.q4_0.gguf",
+  chat_format="functionary-v2",
+  tokenizer=LlamaHFTokenizer.from_pretrained("meetkai/functionary-small-v2.2-GGUF")
+)
+```
+</details>
+
 ### Multi-modal Models
 
 `llama-cpp-python` supports the llava1.5 family of multi-modal models which allow the language model to

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/README.md

@@ -365,14 +365,10 @@ To constrain the response further to a specific JSON Schema add the schema to th
 
 ### Function Calling
 
-The high-level API also provides a simple interface for function calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format.
-
-The gguf-converted files for functionary can be found here: [functionary-7b-v1](https://huggingface.co/abetlen/functionary-7b-v1-GGUF)
+The high-level API supports OpenAI compatible function and tool calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format.
 
 ```python
 >>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/functionary/llama-model.gguf", chat_format="functionary")
->>> # or 
 >>> llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling")
 >>> llm.create_chat_completion(
       messages = [
@@ -416,6 +412,25 @@ The gguf-converted files for functionary can be found here: [functionary-7b-v1](
 )
 ```
 
+<details>
+<summary>Functionary v2</summary>
+
+The various gguf-converted files for this set of models can be found [here](https://huggingface.co/meetkai). Functionary is able to intelligently call functions and also analyze any provided function outputs to generate coherent responses. All v2 models of functionary supports **parallel function calling**. You can provide either `functionary-v1` or `functionary-v2` for the `chat_format` when initializing the Llama class.
+
+Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is required to provide HF Tokenizer for functionary. The `LlamaHFTokenizer` class can be initialized and passed into the Llama class. This will override the default llama.cpp tokenizer used in Llama class. The tokenizer files are already included in the respective HF repositories hosting the gguf files.
+
+```python
+>>> from llama_cpp import Llama
+>>> from llama_cpp.llama_tokenizer import LlamaHFTokenizer
+>>> llm = Llama.from_pretrained(
+  repo_id="meetkai/functionary-small-v2.2-GGUF",
+  filename="functionary-small-v2.2.q4_0.gguf",
+  chat_format="functionary-v2",
+  tokenizer=LlamaHFTokenizer.from_pretrained("meetkai/functionary-small-v2.2-GGUF")
+)
+```
+</details>
+
 ### Multi-modal Models
 
 `llama-cpp-python` supports the llava1.5 family of multi-modal models which allow the language model to

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/docs/server.md

@@ -76,12 +76,14 @@ Function calling is completely compatible with the OpenAI function calling API a
 
 You'll first need to download one of the available function calling models in GGUF format:
 
-- [functionary-7b-v1](https://huggingface.co/abetlen/functionary-7b-v1-GGUF)
+- [functionary](https://huggingface.co/meetkai)
 
-Then when you run the server you'll need to also specify the `functionary` chat_format
+Then when you run the server you'll need to also specify either `functionary-v1` or `functionary-v2` chat_format.
+
+Note that since functionary requires a HF Tokenizer due to discrepancies between llama.cpp and HuggingFace's tokenizers as mentioned [here](https://github.com/abetlen/llama-cpp-python/blob/main?tab=readme-ov-file#function-calling), you will need to pass in the path to the tokenizer too. The tokenizer files are already included in the respective HF repositories hosting the gguf files.
 
 ```bash
-python3 -m llama_cpp.server --model <model_path> --chat_format functionary
+python3 -m llama_cpp.server --model <model_path_to_functionary_v2_model> --chat_format functionary-v2 --hf_pretrained_model_name_or_path <model_path_to_functionary_v2_tokenizer>
 ```
 
 Check out this [example notebook](https://github.com/abetlen/llama-cpp-python/blob/main/examples/notebooks/Functions.ipynb) for a walkthrough of some interesting use cases for function calling.

llama_cpp_python-0.2.51/examples/gradio_chat/local.py

@@ -0,0 +1,59 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+import gradio as gr
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
+    filename="*q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"),
+    verbose=False
+)
+
+model = "gpt-3.5-turbo"
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+    
+    messages.append({"role": "user", "content": message})
+
+    response = llama.create_chat_completion_openai_v1(
+        model=model,
+        messages=messages,
+        stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(predict, fill_height=True, examples=["What is the capital of France?", "Who was the first person on the moon?"])
+
+
+if __name__ == "__main__":
+    demo.launch()

llama_cpp_python-0.2.51/examples/gradio_chat/server.py

@@ -0,0 +1,56 @@
+import gradio as gr
+
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://localhost:8000/v1",
+    api_key="llama.cpp"
+)
+
+model = "gpt-3.5-turbo"
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+    
+    messages.append({"role": "user", "content": message})
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(predict, fill_height=True, examples=["What is the capital of France?", "Who was the first person on the moon?"])
+
+
+if __name__ == "__main__":
+    demo.launch()

llama_cpp_python-0.2.51/examples/hf_pull/main.py

@@ -0,0 +1,39 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
+    filename="*q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"),
+    verbose=False
+)
+
+response = llama.create_chat_completion(
+    messages=[
+        {
+            "role": "user",
+            "content": "What is the capital of France?"
+        }
+    ],
+    response_format={
+        "type": "json_object",
+        "schema": {
+            "type": "object",
+            "properties": {
+                "country": {"type": "string"},
+                "capital": {"type": "string"}
+            },
+            "required": ["country", "capital"],
+        }
+    },
+    stream=True
+)
+
+for chunk in response:
+    delta = chunk["choices"][0]["delta"]
+    if "content" not in delta:
+        continue
+    print(delta["content"], end="", flush=True)
+
+print()

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/examples/notebooks/Functions.ipynb

@@ -9,7 +9,7 @@
     "The OpenAI compatbile web server in `llama-cpp-python` supports function calling.\n",
     "\n",
     "Function calling allows API clients to specify a schema that gives the model a format it should respond in.\n",
-    "Function calling in `llama-cpp-python` works by combining models pretrained for function calling such as [`functionary`](https://huggingface.co/abetlen/functionary-7b-v1-GGUF) with constrained sampling to produce a response that is compatible with the schema.\n",
+    "Function calling in `llama-cpp-python` works by combining models pretrained for function calling such as [`functionary`](https://huggingface.co/meetkai) with constrained sampling to produce a response that is compatible with the schema.\n",
     "\n",
     "Note however that this improves but does not guarantee that the response will be compatible with the schema.\n",
     "\n",

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.49"
+__version__ = "0.2.51"

{llama_cpp_python-0.2.49 → llama_cpp_python-0.2.51}/llama_cpp/_internals.py

@@ -291,7 +291,7 @@ class _LlamaContext:
 
     def kv_cache_seq_shift(self, seq_id: int, p0: int, p1: int, shift: int):
         assert self.ctx is not None
-        llama_cpp.llama_kv_cache_seq_shift(self.ctx, seq_id, p0, p1, shift)
+        llama_cpp.llama_kv_cache_seq_add(self.ctx, seq_id, p0, p1, shift)
 
     def get_state_size(self) -> int:
         assert self.ctx is not None