llama-cpp-python 0.2.70__tar.gz → 0.2.71__tar.gz

llama_cpp_python-0.2.71/.git/FETCH_HEAD

@@ -0,0 +1 @@
+bf66a283e899e367c82bdf13b89179fb14ed176e		'bf66a283e899e367c82bdf13b89179fb14ed176e' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.71/.git/HEAD

@@ -0,0 +1 @@
+bf66a283e899e367c82bdf13b89179fb14ed176e

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1VKZzRtOWNLdzR0dno0b0Z5M1U4QkhFd1NrRGI3NTFSSWpZTw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX0NvN3dnY0E3NGZPc1JRODZ5WmtFTm1MR2U3R3M2bjFRVVByWg==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.71/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 bf66a283e899e367c82bdf13b89179fb14ed176e runner <runner@fv-az979-264.(none)> 1715238244 +0000	checkout: moving from master to refs/tags/v0.2.71

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/FETCH_HEAD

@@ -0,0 +1 @@
+911b3900dded9a1cfe0f0e41b82c7a29baf3a217		'911b3900dded9a1cfe0f0e41b82c7a29baf3a217' of https://github.com/ggerganov/llama.cpp

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+911b3900dded9a1cfe0f0e41b82c7a29baf3a217

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1VKZzRtOWNLdzR0dno0b0Z5M1U4QkhFd1NrRGI3NTFSSWpZTw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX0NvN3dnY0E3NGZPc1JRODZ5WmtFTm1MR2U3R3M2bjFRVVByWg==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 4426e2987b566f09c7aa96ada9706cc778637620 runner <runner@fv-az979-264.(none)> 1715238247 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+4426e2987b566f09c7aa96ada9706cc778637620 911b3900dded9a1cfe0f0e41b82c7a29baf3a217 runner <runner@fv-az979-264.(none)> 1715238248 +0000	checkout: moving from master to 911b3900dded9a1cfe0f0e41b82c7a29baf3a217

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 4426e2987b566f09c7aa96ada9706cc778637620 runner <runner@fv-az979-264.(none)> 1715238247 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 4426e2987b566f09c7aa96ada9706cc778637620 runner <runner@fv-az979-264.(none)> 1715238247 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1VKZzRtOWNLdzR0dno0b0Z5M1U4QkhFd1NrRGI3NTFSSWpZTw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX0NvN3dnY0E3NGZPc1JRODZ5WmtFTm1MR2U3R3M2bjFRVVByWg==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 c339310f6ff914c5b94fb2353f01a33dfc35f64f runner <runner@fv-az979-264.(none)> 1715238249 +0000	clone: from https://github.com/nomic-ai/kompute.git
+c339310f6ff914c5b94fb2353f01a33dfc35f64f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az979-264.(none)> 1715238250 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 c339310f6ff914c5b94fb2353f01a33dfc35f64f runner <runner@fv-az979-264.(none)> 1715238249 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 c339310f6ff914c5b94fb2353f01a33dfc35f64f runner <runner@fv-az979-264.(none)> 1715238249 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/modules/kompute/objects/31/01a2b17b7518586fcd800d9eae67102fd9cb2b

@@ -0,0 +1,3 @@
+x
�TMo�@�_1�s��i�^�����‡�G՞V{e����*��vUF��x�
�̛�o�̺�������)�+���)V
+�v�U+r�r	F��[��,��)LG����y)*Qv%���3�+B������z�Q���t"�u'��6�P*�e]��;�"X%�3M�����q�ϱ�X<�bX;Ӏ�CW쒊�uو�O��;�a�~�vE��C��>͏�ӈm��.W��`�k&D��bN��i��J��z��H�K$�P��`
+�^�'=�(��wR�2x Q��h'Y��N�u��T&U���IU������"��m�W�L׸g܌��y$���wn��ӧ��ID��]��,8f���`� ��G.r��<GInz�:`�~�l�>�^��������;vq!����(�p3
+�N��U*�Z&�Q�5�1��-`r��/Nz7\wF�8�~��Dn��J�.C�y���@5J��|<,"���Ns$*�&4J�u��?�~I��5"X,�h��~�����pn{�[���%ʎkz>���

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/modules/kompute/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+c339310f6ff914c5b94fb2353f01a33dfc35f64f refs/remotes/origin/master

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/modules/kompute/refs/heads/master

@@ -0,0 +1 @@
+c339310f6ff914c5b94fb2353f01a33dfc35f64f

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/.git/modules/vendor/llama.cpp/modules/kompute/shallow

@@ -1,2 +1,2 @@
 4565194ed7c32d1d2efa32ceab4d3c6cae006306
-d1e3b0953cf66acc94b2e29693e221427b2c1f3f
+c339310f6ff914c5b94fb2353f01a33dfc35f64f

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/objects/55/20ba732ffcafeca43b5bbe635cee7dfd4f6040

@@ -0,0 +1,2 @@
+x
5�K
+� @�Ʈ����^2��?�&�=���g�=G�|u�%�Q�`�|!�S<h�["pXfV��	�Q�J6���N����j���J�o9�o0�z

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+4426e2987b566f09c7aa96ada9706cc778637620 refs/remotes/origin/master

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+4426e2987b566f09c7aa96ada9706cc778637620

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/refs/tags/b2824

@@ -0,0 +1 @@
+4426e2987b566f09c7aa96ada9706cc778637620

llama_cpp_python-0.2.71/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1,2 @@
+4426e2987b566f09c7aa96ada9706cc778637620
+911b3900dded9a1cfe0f0e41b82c7a29baf3a217

llama_cpp_python-0.2.71/.git/refs/tags/v0.2.71

@@ -0,0 +1 @@
+bf66a283e899e367c82bdf13b89179fb14ed176e

llama_cpp_python-0.2.71/.git/shallow

@@ -0,0 +1 @@
+bf66a283e899e367c82bdf13b89179fb14ed176e

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/CHANGELOG.md

@@ -7,9 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.71]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@911b3900dded9a1cfe0f0e41b82c7a29baf3a217
+- fix: Make leading bos_token optional for image chat formats, fix nanollava system message by @abetlen in 77122638b4153e31d9f277b3d905c2900b536632
+- fix: free last image embed in llava chat handler by @abetlen in 3757328b703b2cd32dcbd5853271e3a8c8599fe7
+
 ## [0.2.70]
 
-- feat: Update llama.cpp to ggerganov/llama.cpp@
+- feat: Update llama.cpp to ggerganov/llama.cpp@c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1
 - feat: fill-in-middle support by @CISC in #1386
 - fix: adding missing args in create_completion for functionary chat handler by @skalade in #1430
 - docs: update README.md @eltociear in #1432

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.70
+Version: 0.2.71
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.70"
+__version__ = "0.2.71"

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/llama_cpp/llama_chat_format.py

@@ -2603,13 +2603,23 @@ class Llava15ChatHandler:
 
         image_urls = self.get_image_urls(messages)
         template = jinja2.Template(self.CHAT_FORMAT)
-        text = template.render(messages=messages, add_generation_prompt=True)
+        text = template.render(
+            messages=messages,
+            add_generation_prompt=True,
+            eos_token=llama.detokenize([llama.token_eos()]),
+            bos_token=llama.detokenize([llama.token_bos()]),
+        )
         split_text = self.split_text_on_image_urls(text, image_urls)
 
         def embed_image_bytes(image_bytes: bytes):
             if self._last_image_embed is not None and self._last_image_hash is not None and hash(image_bytes) == self._last_image_hash:
                 return self._last_image_embed
             with suppress_stdout_stderr(disable=self.verbose):
+                # Free the previous image embed
+                if self._last_image_embed is not None:
+                    self._llava_cpp.llava_image_embed_free(self._last_image_embed)
+                    self._last_image_embed = None
+                    self._last_image_hash = None
                 embed = (
                     self._llava_cpp.llava_image_embed_make_with_bytes(
                         self.clip_ctx,
@@ -2624,9 +2634,9 @@ class Llava15ChatHandler:
 
         # Evaluate prompt
         llama.reset()
-        for i, (type_, value) in enumerate(split_text):
+        for type_, value in split_text:
             if type_ == "text":
-                tokens = llama.tokenize(value.encode("utf8"), add_bos=i == 0)
+                tokens = llama.tokenize(value.encode("utf8"), add_bos=False, special=True)
                 if llama.n_tokens + len(tokens) > llama.n_ctx():
                     raise ValueError("Prompt exceeds n_ctx") # TODO: Fix
                 llama.eval(tokens)
@@ -2644,6 +2654,8 @@ class Llava15ChatHandler:
                         llama.n_batch,
                         n_past_p,
                     )
+                # Required to avoid issues with hf tokenizer
+                llama.input_ids[llama.n_tokens : n_past.value] = -1
                 llama.n_tokens = n_past.value
 
         # Get prompt tokens to avoid a cache miss
@@ -3033,6 +3045,7 @@ class NanoLlavaChatHandler(Llava15ChatHandler):
     # Answer the question<|im_end|><|im_start|>user
     # <image>
     # What is the picture about?<|im_end|><|im_start|>assistant
+    DEFAULT_SYSTEM_MESSAGE = "Answer the question"
 
     CHAT_FORMAT = (
         "{% for message in messages %}"

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/llama_cpp/llama_cpp.py

@@ -296,7 +296,9 @@ LLAMA_VOCAB_TYPE_WPM = 3
 #     LLAMA_VOCAB_PRE_TYPE_GPT2           = 7,
 #     LLAMA_VOCAB_PRE_TYPE_REFACT         = 8,
 #     LLAMA_VOCAB_PRE_TYPE_COMMAND_R      = 9,
-#     LLAMA_VOCAB_PRE_TYPE_OLMO           = 10,
+#     LLAMA_VOCAB_PRE_TYPE_QWEN2          = 10,
+#     LLAMA_VOCAB_PRE_TYPE_OLMO           = 11,
+#     LLAMA_VOCAB_PRE_TYPE_DBRX           = 12,
 # };
 LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0
 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1
@@ -308,7 +310,9 @@ LLAMA_VOCAB_PRE_TYPE_STARCODER = 6
 LLAMA_VOCAB_PRE_TYPE_GPT2 = 7
 LLAMA_VOCAB_PRE_TYPE_REFACT = 8
 LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9
-LLAMA_VOCAB_PRE_TYPE_OLMO = 10
+LLAMA_VOCAB_PRE_TYPE_QWEN2 = 10
+LLAMA_VOCAB_PRE_TYPE_OLMO = 11
+LLAMA_VOCAB_PRE_TYPE_DBRX = 12
 
 
 # // note: these values should be synchronized with ggml_rope
@@ -377,6 +381,7 @@ LLAMA_TOKEN_TYPE_BYTE = 6
 #     LLAMA_FTYPE_MOSTLY_IQ2_M         = 29, // except 1d tensors
 #     LLAMA_FTYPE_MOSTLY_IQ4_XS        = 30, // except 1d tensors
 #     LLAMA_FTYPE_MOSTLY_IQ1_M         = 31, // except 1d tensors
+#     LLAMA_FTYPE_MOSTLY_BF16          = 32, // except 1d tensors
 
 #     LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
 # };
@@ -409,6 +414,8 @@ LLAMA_FTYPE_MOSTLY_IQ3_M = 27
 LLAMA_FTYPE_MOSTLY_IQ2_S = 28
 LLAMA_FTYPE_MOSTLY_IQ2_M = 29
 LLAMA_FTYPE_MOSTLY_IQ4_XS = 30
+LLAMA_FTYPE_MOSTLY_IQ1_M = 31
+LLAMA_FTYPE_MOSTLY_BF16 = 32
 LLAMA_FTYPE_GUESSED = 1024
 
 # enum llama_rope_scaling_type {

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/common/build-info.cpp

@@ -1,4 +1,4 @@
 int LLAMA_BUILD_NUMBER = 1;
-char const *LLAMA_COMMIT = "c0e6fbf";
+char const *LLAMA_COMMIT = "911b390";
 char const *LLAMA_COMPILER = "cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0";
 char const *LLAMA_BUILD_TARGET = "x86_64-linux-gnu";

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/convert-hf-to-gguf-update.py

@@ -67,7 +67,9 @@ models = [
     {"name": "gpt-2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
     {"name": "refact",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },
     {"name": "command-r",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
+    {"name": "qwen2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
     {"name": "olmo",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
+    {"name": "dbrx",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },
 ]
 
 # make directory "models/tokenizers" if it doesn't exist
@@ -151,6 +153,8 @@ for model in models:
     # print the "pre_tokenizer" content from the tokenizer.json
     with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
         cfg = json.load(f)
+        normalizer = cfg["normalizer"]
+        logger.info("normalizer: " + json.dumps(normalizer, indent=4))
         pre_tokenizer = cfg["pre_tokenizer"]
         logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
 

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/convert-hf-to-gguf.py

@@ -314,9 +314,15 @@ class Model(ABC):
         if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8":
             # ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01
             res = "command-r"
+        if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
+            # ref: https://huggingface.co/Qwen/Qwen1.5-7B
+            res = "qwen2"
         if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166":
             # ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf
             res = "olmo"
+        if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e":
+            # ref: https://huggingface.co/databricks/dbrx-instruct
+            res = "dbrx"
 
         if res is None:
             logger.warning("\n")

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/convert.py

@@ -1508,25 +1508,27 @@ def main(args_in: list[str] | None = None) -> None:
     if args.big_endian:
         endianess = gguf.GGUFEndian.BIG
 
-    params = Params.load(model_plus)
-    if params.n_ctx == -1:
-        if args.ctx is None:
-            msg = """\
-                The model doesn't have a context size, and you didn't specify one with --ctx
-                Please specify one with --ctx:
-                 - LLaMA v1: --ctx 2048
-                 - LLaMA v2: --ctx 4096"""
-            parser.error(textwrap.dedent(msg))
-        params.n_ctx = args.ctx
-
-    if args.outtype:
-        params.ftype = {
-            "f32": GGMLFileType.AllF32,
-            "f16": GGMLFileType.MostlyF16,
-            "q8_0": GGMLFileType.MostlyQ8_0,
-        }[args.outtype]
-
-    logger.info(f"params = {params}")
+    params = None
+    if args.pad_vocab or not args.vocab_only:
+        params = Params.load(model_plus)
+        if params.n_ctx == -1:
+            if args.ctx is None:
+                msg = """\
+                    The model doesn't have a context size, and you didn't specify one with --ctx
+                    Please specify one with --ctx:
+                     - LLaMA v1: --ctx 2048
+                     - LLaMA v2: --ctx 4096"""
+                parser.error(textwrap.dedent(msg))
+            params.n_ctx = args.ctx
+
+        if args.outtype:
+            params.ftype = {
+                "f32": GGMLFileType.AllF32,
+                "f16": GGMLFileType.MostlyF16,
+                "q8_0": GGMLFileType.MostlyQ8_0,
+            }[args.outtype]
+
+        logger.info(f"params = {params}")
 
     model_parent_path = model_plus.paths[0].parent
     vocab_path = Path(args.vocab_dir or args.model or model_parent_path)
@@ -1539,6 +1541,17 @@ def main(args_in: list[str] | None = None) -> None:
         if not args.outfile:
             raise ValueError("need --outfile if using --vocab-only")
         outfile = args.outfile
+        if params is None:
+            params = Params(
+                n_vocab    = vocab.vocab_size,
+                n_embd     = 1,
+                n_layer    = 1,
+                n_ctx      = 1,
+                n_ff       = 1,
+                n_head     = 1,
+                n_head_kv  = 1,
+                f_norm_eps = 1e-5,
+            )
         OutputFile.write_vocab_only(outfile, params, vocab, special_vocab,
                                     endianess=endianess, pad_vocab=args.pad_vocab)
         logger.info(f"Wrote {outfile}")

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/examples/finetune/finetune.cpp

@@ -575,7 +575,7 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs(
     GGML_ASSERT(tokens_input->type == GGML_TYPE_I32);
 
     auto add_to_f32 = [] (struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b) {
-        if (ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16) {
+        if (ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16) {
             return ggml_add_cast(ctx, a, b, GGML_TYPE_F32);
         } else if (a->type == GGML_TYPE_F32) {
             return ggml_add(ctx, a, b);

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/examples/quantize/quantize.cpp

@@ -46,7 +46,8 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
     { "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 4.45G, +0.0122 ppl @ LLaMA-v1-7B", },
     { "Q6_K",   LLAMA_FTYPE_MOSTLY_Q6_K,   " 5.15G, +0.0008 ppl @ LLaMA-v1-7B", },
     { "Q8_0",   LLAMA_FTYPE_MOSTLY_Q8_0,   " 6.70G, +0.0004 ppl @ LLaMA-v1-7B", },
-    { "F16",    LLAMA_FTYPE_MOSTLY_F16,    "13.00G              @ 7B", },
+    { "F16",    LLAMA_FTYPE_MOSTLY_F16,    "14.00G, -0.0020 ppl @ Mistral-7B", },
+    { "BF16",   LLAMA_FTYPE_MOSTLY_BF16,   "14.00G, -0.0050 ppl @ Mistral-7B", },
     { "F32",    LLAMA_FTYPE_ALL_F32,       "26.00G              @ 7B", },
     // Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.
     { "COPY",   LLAMA_FTYPE_ALL_F32,       "only copy tensors, no quantizing", },

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/examples/server/README.md

@@ -331,7 +331,7 @@ Notice that each `probs` is an array of length `n_probs`.
 
     `content`: Set the text to tokenize.
 
-    Note that a special `BOS` token is never inserted.
+    `add_special`: Boolean indicating if special tokens, i.e. `BOS`, should be inserted.  Default: `false`
 
 - **POST** `/detokenize`: Convert tokens to text.
 

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/examples/server/server.cpp

@@ -3647,7 +3647,8 @@ int main(int argc, char ** argv) {
 
         std::vector<llama_token> tokens;
         if (body.count("content") != 0) {
-            tokens = ctx_server.tokenize(body["content"], false);
+            const bool add_special = json_value(body, "add_special", false);
+            tokens = ctx_server.tokenize(body["content"], add_special);
         }
         const json data = format_tokenizer_response(tokens);
         return res.set_content(data.dump(), "application/json; charset=utf-8");

{llama_cpp_python-0.2.70 → llama_cpp_python-0.2.71}/vendor/llama.cpp/examples/server/tests/features/server.feature

@@ -7,6 +7,7 @@ Feature: llama.cpp server
     And   a model file tinyllamas/stories260K.gguf from HF repo ggml-org/models
     And   a model file test-model.gguf
     And   a model alias tinyllama-2
+    And   BOS token is 1
     And   42 as server seed
       # KV Cache corresponds to the total amount of tokens
       # that can be stored across all independent sequences: #4130
@@ -91,7 +92,18 @@ Feature: llama.cpp server
     """
     What is the capital of France ?
     """
-    Then tokens can be detokenize
+    Then tokens can be detokenized
+    And  tokens do not begin with BOS
+
+  Scenario: Tokenize w/ BOS
+    Given adding special tokens
+    When  tokenizing:
+    """
+    What is the capital of Germany?
+    """
+    Then  tokens begin with BOS
+    Given first token is removed
+    Then  tokens can be detokenized
 
   Scenario: Models available
     Given available models