llama-cpp-python 0.2.52__tar.gz → 0.2.54__tar.gz

llama_cpp_python-0.2.54/.git/FETCH_HEAD

@@ -0,0 +1 @@
+d5df431278433b580e52222dbf4174f5102585b1		'd5df431278433b580e52222dbf4174f5102585b1' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.54/.git/HEAD

@@ -0,0 +1 @@
+d5df431278433b580e52222dbf4174f5102585b1

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX01MZUJWbTRYUE96TmlVaWxQNFRMQ0RBeEtWVFdycjNsdUI5NQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzdpQXRKODgxbmFTbmVlTHlEM0pqN0pCRE1mazRldzBySUlxMw==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.54/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d5df431278433b580e52222dbf4174f5102585b1 runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316972 +0000	checkout: moving from master to refs/tags/v0.2.54

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/FETCH_HEAD

@@ -0,0 +1 @@
+c2224f003bf9cf558b1a3c57033563e11a4de9a5		'c2224f003bf9cf558b1a3c57033563e11a4de9a5' of https://github.com/ggerganov/llama.cpp

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+c2224f003bf9cf558b1a3c57033563e11a4de9a5

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX01MZUJWbTRYUE96TmlVaWxQNFRMQ0RBeEtWVFdycjNsdUI5NQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzdpQXRKODgxbmFTbmVlTHlEM0pqN0pCRE1mazRldzBySUlxMw==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 38d16b142624bdd7c41d9955752b7f7b59c5e048 runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316973 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+38d16b142624bdd7c41d9955752b7f7b59c5e048 c2224f003bf9cf558b1a3c57033563e11a4de9a5 runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316974 +0000	checkout: moving from master to c2224f003bf9cf558b1a3c57033563e11a4de9a5

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 38d16b142624bdd7c41d9955752b7f7b59c5e048 runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316973 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 38d16b142624bdd7c41d9955752b7f7b59c5e048 runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316973 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX01MZUJWbTRYUE96TmlVaWxQNFRMQ0RBeEtWVFdycjNsdUI5NQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzdpQXRKODgxbmFTbmVlTHlEM0pqN0pCRE1mazRldzBySUlxMw==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316974 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316975 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316974 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1240-729.ompflkn1s5uuratdiiqak2yqtf.ex.internal.cloudapp.net> 1709316974 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+38d16b142624bdd7c41d9955752b7f7b59c5e048 refs/remotes/origin/master

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+38d16b142624bdd7c41d9955752b7f7b59c5e048

llama_cpp_python-0.2.54/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1,2 @@
+38d16b142624bdd7c41d9955752b7f7b59c5e048
+c2224f003bf9cf558b1a3c57033563e11a4de9a5

llama_cpp_python-0.2.54/.git/refs/tags/v0.2.54

@@ -0,0 +1 @@
+d5df431278433b580e52222dbf4174f5102585b1

llama_cpp_python-0.2.54/.git/shallow

@@ -0,0 +1 @@
+d5df431278433b580e52222dbf4174f5102585b1

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.54]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@cb49e0f8c906e5da49e9f6d64a57742a9a241c6a
+- docs: fix typo in README.md embeddings example by @iamlemec in #1232
+
+## [0.2.53]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@cb49e0f8c906e5da49e9f6d64a57742a9a241c6a
+- fix: eos/bos_token set correctly for Jinja2ChatFormatter and automatic chat formatter by @CISC in #1230
+
 ## [0.2.52]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@a33e6a0d2a66104ea9a906bdbf8a94d050189d91

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.52
+Version: 0.2.54
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -329,7 +329,16 @@ By default [`from_pretrained`](https://llama-cpp-python.readthedocs.io/en/latest
 
 The high-level API also provides a simple interface for chat completion.
 
-Note that `chat_format` option must be set for the particular model you are using.
+Chat completion requires that the model know how to format the messages into a single prompt.
+The `Llama` class does this using pre-registered chat formats (ie. `chatml`, `llama-2`, `gemma`, etc) or by providing a custom chat handler object.
+
+The model will will format the messages into a single prompt using the following order of precedence:
+  - Use the `chat_handler` if provided
+  - Use the `chat_format` if provided
+  - Use the `tokenizer.chat_template` from the `gguf` model's metadata (should work for most new models, older models may not have this)
+  - else, fallback to the `llama-2` chat format
+
+Set `verbose=True` to see the selected chat format.
 
 ```python
 >>> from llama_cpp import Llama
@@ -511,6 +520,38 @@ Then you'll need to use a custom chat handler to load the clip model and process
 )
 ```
 
+<details>
+<summary>Loading a Local Image</summary>
+
+Images can be passed as base64 encoded data URIs. The following example demonstrates how to do this.
+
+```python
+import base64
+
+def image_to_base64_data_uri(file_path):
+    with open(file_path, "rb") as img_file:
+        base64_data = base64.b64encode(img_file.read()).decode('utf-8')
+        return f"data:image/png;base64,{base64_data}"
+
+# Replace 'file_path.png' with the actual path to your PNG file
+file_path = 'file_path.png'
+data_uri = image_to_base64_data_uri(file_path)
+
+messages = [
+    {"role": "system", "content": "You are an assistant who perfectly describes images."},
+    {
+        "role": "user",
+        "content": [
+            {"type": "image_url", "image_url": {"url": data_uri }},
+            {"type" : "text", "text": "Describe this image in detail please."}
+        ]
+    }
+]
+
+```
+
+</details>
+
 ### Speculative Decoding
 
 `llama-cpp-python` supports speculative decoding which allows the model to generate completions based on a draft model.
@@ -536,7 +577,7 @@ To generate text embeddings use [`create_embedding`](http://localhost:8000/api-r
 ```python
 import llama_cpp
 
-llm = llama_cpp.Llama(model_path="path/to/model.gguf", embeddings=True)
+llm = llama_cpp.Llama(model_path="path/to/model.gguf", embedding=True)
 
 embeddings = llm.create_embedding("Hello, world!")
 
@@ -588,6 +629,12 @@ python3 -m llama_cpp.server --model models/7B/llama-model.gguf --chat_format cha
 That will format the prompt according to how model expects it. You can find the prompt format in the model card.
 For possible options, see [llama_cpp/llama_chat_format.py](llama_cpp/llama_chat_format.py) and look for lines starting with "@register_chat_format".
 
+If you have `huggingface-hub` installed, you can also use the `--hf_model_repo_id` flag to load a model from the Hugging Face Hub.
+
+```bash
+python3 -m llama_cpp.server --hf_model_repo_id Qwen/Qwen1.5-0.5B-Chat-GGUF --model '*q8_0.gguf'
+```
+
 ### Web Server Features
 
 - [Local Copilot replacement](https://llama-cpp-python.readthedocs.io/en/latest/server/#code-completion)

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/README.md

@@ -286,7 +286,16 @@ By default [`from_pretrained`](https://llama-cpp-python.readthedocs.io/en/latest
 
 The high-level API also provides a simple interface for chat completion.
 
-Note that `chat_format` option must be set for the particular model you are using.
+Chat completion requires that the model know how to format the messages into a single prompt.
+The `Llama` class does this using pre-registered chat formats (ie. `chatml`, `llama-2`, `gemma`, etc) or by providing a custom chat handler object.
+
+The model will will format the messages into a single prompt using the following order of precedence:
+  - Use the `chat_handler` if provided
+  - Use the `chat_format` if provided
+  - Use the `tokenizer.chat_template` from the `gguf` model's metadata (should work for most new models, older models may not have this)
+  - else, fallback to the `llama-2` chat format
+
+Set `verbose=True` to see the selected chat format.
 
 ```python
 >>> from llama_cpp import Llama
@@ -468,6 +477,38 @@ Then you'll need to use a custom chat handler to load the clip model and process
 )
 ```
 
+<details>
+<summary>Loading a Local Image</summary>
+
+Images can be passed as base64 encoded data URIs. The following example demonstrates how to do this.
+
+```python
+import base64
+
+def image_to_base64_data_uri(file_path):
+    with open(file_path, "rb") as img_file:
+        base64_data = base64.b64encode(img_file.read()).decode('utf-8')
+        return f"data:image/png;base64,{base64_data}"
+
+# Replace 'file_path.png' with the actual path to your PNG file
+file_path = 'file_path.png'
+data_uri = image_to_base64_data_uri(file_path)
+
+messages = [
+    {"role": "system", "content": "You are an assistant who perfectly describes images."},
+    {
+        "role": "user",
+        "content": [
+            {"type": "image_url", "image_url": {"url": data_uri }},
+            {"type" : "text", "text": "Describe this image in detail please."}
+        ]
+    }
+]
+
+```
+
+</details>
+
 ### Speculative Decoding
 
 `llama-cpp-python` supports speculative decoding which allows the model to generate completions based on a draft model.
@@ -493,7 +534,7 @@ To generate text embeddings use [`create_embedding`](http://localhost:8000/api-r
 ```python
 import llama_cpp
 
-llm = llama_cpp.Llama(model_path="path/to/model.gguf", embeddings=True)
+llm = llama_cpp.Llama(model_path="path/to/model.gguf", embedding=True)
 
 embeddings = llm.create_embedding("Hello, world!")
 
@@ -545,6 +586,12 @@ python3 -m llama_cpp.server --model models/7B/llama-model.gguf --chat_format cha
 That will format the prompt according to how model expects it. You can find the prompt format in the model card.
 For possible options, see [llama_cpp/llama_chat_format.py](llama_cpp/llama_chat_format.py) and look for lines starting with "@register_chat_format".
 
+If you have `huggingface-hub` installed, you can also use the `--hf_model_repo_id` flag to load a model from the Hugging Face Hub.
+
+```bash
+python3 -m llama_cpp.server --hf_model_repo_id Qwen/Qwen1.5-0.5B-Chat-GGUF --model '*q8_0.gguf'
+```
+
 ### Web Server Features
 
 - [Local Copilot replacement](https://llama-cpp-python.readthedocs.io/en/latest/server/#code-completion)

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.52"
+__version__ = "0.2.54"

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/llama_cpp/_internals.py

@@ -357,21 +357,6 @@ class _LlamaContext:
             penalty_present,
         )
 
-    def sample_classifier_free_guidance(
-        self,
-        candidates: "_LlamaTokenDataArray",
-        guidance_ctx: "_LlamaContext",
-        scale: float,
-    ):
-        assert self.ctx is not None
-        assert guidance_ctx.ctx is not None
-        llama_cpp.llama_sample_classifier_free_guidance(
-            self.ctx,
-            llama_cpp.byref(candidates.candidates),
-            guidance_ctx.ctx,
-            scale,
-        )
-
     def sample_softmax(self, candidates: "_LlamaTokenDataArray"):
         assert self.ctx is not None
         llama_cpp.llama_sample_softmax(
@@ -720,7 +705,7 @@ class _LlamaSamplingContext:
         return ctx_main.model.detokenize(self.prev[-n:]).decode("utf-8")
 
     def sample(
-        self, ctx_main: _LlamaContext, ctx_cfg: Optional[_LlamaContext] = None, idx: int = 0, logits_array: Optional[npt.NDArray[np.single]] = None
+        self, ctx_main: _LlamaContext, idx: int = 0, logits_array: Optional[npt.NDArray[np.single]] = None
     ):
         n_vocab = ctx_main.model.n_vocab()
         id: int = 0
@@ -741,11 +726,6 @@ class _LlamaSamplingContext:
         )  # TODO: Only create this once
         token_data_array.copy_logits(logits_array)
 
-        if ctx_cfg is not None:
-            ctx_main.sample_classifier_free_guidance(
-                token_data_array, ctx_cfg, self.params.cfg_scale
-            )
-
         # apply penalties
         if len(self.prev) > 0:
             nl_token = ctx_main.model.token_nl()

{llama_cpp_python-0.2.52 → llama_cpp_python-0.2.54}/llama_cpp/llama.py

@@ -86,7 +86,6 @@ class Llama:
         yarn_beta_fast: float = 32.0,
         yarn_beta_slow: float = 1.0,
         yarn_orig_ctx: int = 0,
-        mul_mat_q: bool = True,
         logits_all: bool = False,
         embedding: bool = False,
         offload_kqv: bool = True,
@@ -291,7 +290,6 @@ class Llama:
             yarn_beta_slow if yarn_beta_slow != 0.0 else 0
         )
         self.context_params.yarn_orig_ctx = yarn_orig_ctx if yarn_orig_ctx != 0 else 0
-        self.context_params.mul_mat_q = mul_mat_q
         self.context_params.logits_all = (
             logits_all if draft_model is None else True
         )  # Must be set to True for speculative decoding
@@ -408,11 +406,11 @@ class Llama:
                 except:
                     bos_token_id = self.token_bos()
 
-                eos_token = self.detokenize([eos_token_id]).decode("utf-8")
-                bos_token = self.detokenize([bos_token_id]).decode("utf-8")
+                eos_token = self._model.token_get_text(eos_token_id)
+                bos_token = self._model.token_get_text(bos_token_id)
 
                 if self.verbose:
-                    print(f"Using chat template: {template}", file=sys.stderr)
+                    print(f"Using gguf chat template: {template}", file=sys.stderr)
                     print(f"Using chat eos_token: {eos_token}", file=sys.stderr)
                     print(f"Using chat bos_token: {bos_token}", file=sys.stderr)
 
@@ -422,6 +420,8 @@ class Llama:
 
         if self.chat_format is None and self.chat_handler is None:
             self.chat_format = "llama-2"
+            if self.verbose:
+                print(f"Using fallback chat format: {chat_format}", file=sys.stderr)
 
     @property
     def ctx(self) -> llama_cpp.llama_context_p:
@@ -1724,7 +1724,6 @@ class Llama:
             yarn_beta_fast=self.context_params.yarn_beta_fast,
             yarn_beta_slow=self.context_params.yarn_beta_slow,
             yarn_orig_ctx=self.context_params.yarn_orig_ctx,
-            mul_mat_q=self.context_params.mul_mat_q,
             logits_all=self.context_params.logits_all,
             embedding=self.context_params.embedding,
             # Sampling Params
@@ -1768,7 +1767,6 @@ class Llama:
             yarn_beta_fast=state["yarn_beta_fast"],
             yarn_beta_slow=state["yarn_beta_slow"],
             yarn_orig_ctx=state["yarn_orig_ctx"],
-            mul_mat_q=state["mul_mat_q"],
             logits_all=state["logits_all"],
             embedding=state["embedding"],
             # Sampling Params