llama-cpp-python 0.2.69__tar.gz → 0.2.70__tar.gz

llama_cpp_python-0.2.70/.git/FETCH_HEAD

@@ -0,0 +1 @@
+9ce5cb376a12a56028aec1fd3b0edc55949b996f		'9ce5cb376a12a56028aec1fd3b0edc55949b996f' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.70/.git/HEAD

@@ -0,0 +1 @@
+9ce5cb376a12a56028aec1fd3b0edc55949b996f

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3k2Y2NrSEplV2ZpcDZGZnR6VldrRzlDQVBhQUdIYTNaOTJxNQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1VKZzRtOWNLdzR0dno0b0Z5M1U4QkhFd1NrRGI3NTFSSWpZTw==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.70/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 9ce5cb376a12a56028aec1fd3b0edc55949b996f runner <runner@fv-az564-924.(none)> 1715150262 +0000	checkout: moving from master to refs/tags/v0.2.70

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/FETCH_HEAD

@@ -0,0 +1 @@
+c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1		'c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1' of https://github.com/ggerganov/llama.cpp

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3k2Y2NrSEplV2ZpcDZGZnR6VldrRzlDQVBhQUdIYTNaOTJxNQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1VKZzRtOWNLdzR0dno0b0Z5M1U4QkhFd1NrRGI3NTFSSWpZTw==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 3855416027cb25d9a708ffa5581cf503a87856a6 runner <runner@fv-az564-924.(none)> 1715150263 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+3855416027cb25d9a708ffa5581cf503a87856a6 c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1 runner <runner@fv-az564-924.(none)> 1715150264 +0000	checkout: moving from master to c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 3855416027cb25d9a708ffa5581cf503a87856a6 runner <runner@fv-az564-924.(none)> 1715150263 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 3855416027cb25d9a708ffa5581cf503a87856a6 runner <runner@fv-az564-924.(none)> 1715150263 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3k2Y2NrSEplV2ZpcDZGZnR6VldrRzlDQVBhQUdIYTNaOTJxNQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1VKZzRtOWNLdzR0dno0b0Z5M1U4QkhFd1NrRGI3NTFSSWpZTw==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az564-924.(none)> 1715150265 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az564-924.(none)> 1715150266 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az564-924.(none)> 1715150265 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az564-924.(none)> 1715150265 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/objects/a9/eee7cfc8f5eef59c4d4fa805ad3bbe86ea86d8

@@ -0,0 +1 @@
+x
+)JMU064e040031Q�����,���+�d���t��i��m|�Yv�]�,9?��$1����_YO�f��7j�l_}��P�u�����4��T���MME��>���o՘��II|QjbJjH��V�Ѯ,�6�O_�w�ي�ճ�U�e�@T�<��P���A�o���M�ܾ���P��z%��)��>����5g�n�u�Q7=�	�US��W�_��XP�����i)���oWed_-ޡ7i�U������$���U���{�:sk��|�A�۞�
���

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+3855416027cb25d9a708ffa5581cf503a87856a6 refs/remotes/origin/master

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+3855416027cb25d9a708ffa5581cf503a87856a6

llama_cpp_python-0.2.70/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1,2 @@
+3855416027cb25d9a708ffa5581cf503a87856a6
+c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1

llama_cpp_python-0.2.70/.git/refs/tags/v0.2.70

@@ -0,0 +1 @@
+9ce5cb376a12a56028aec1fd3b0edc55949b996f

llama_cpp_python-0.2.70/.git/shallow

@@ -0,0 +1 @@
+9ce5cb376a12a56028aec1fd3b0edc55949b996f

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/.github/dependabot.yml

@@ -8,8 +8,12 @@ updates:
   - package-ecosystem: "pip" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "daily"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "weekly"    
+      interval: "daily"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"   

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.70]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@
+- feat: fill-in-middle support by @CISC in #1386
+- fix: adding missing args in create_completion for functionary chat handler by @skalade in #1430
+- docs: update README.md @eltociear in #1432
+- fix: chat_format log where auto-detected format prints None by @balvisio in #1434
+- feat(server): Add support for setting root_path by @abetlen in 0318702cdc860999ee70f277425edbbfe0e60419
+- feat(ci): Add docker checks and check deps more frequently by @Smartappli in #1426
+- fix: detokenization case where first token does not start with a leading space by @noamgat in #1375
+- feat: Implement streaming for Functionary v2 + Bug fixes by @jeffrey-fong in #1419
+- fix: Use memmove to copy str_value kv_override by @abetlen in 9f7a85571ae80d3b6ddbd3e1bae407b9f1e3448a
+- feat(server): Remove temperature bounds checks for server by @abetlen in 0a454bebe67d12a446981eb16028c168ca5faa81
+- fix(server): Propagate flash_attn to model load by @dthuerck in #1424
+
 ## [0.2.69]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@6ecf3189e00a1e8e737a78b6d10e1d7006e050a2

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.69
+Version: 0.2.70
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -560,7 +560,7 @@ chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin")
 llm = Llama(
   model_path="./path/to/llava/llama-model.gguf",
   chat_handler=chat_handler,
-  n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
+  n_ctx=2048, # n_ctx should be increased to accommodate the image embedding
 )
 llm.create_chat_completion(
     messages = [
@@ -591,7 +591,7 @@ llm = Llama.from_pretrained(
   repo_id="vikhyatk/moondream2",
   filename="*text-model*",
   chat_handler=chat_handler,
-  n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
+  n_ctx=2048, # n_ctx should be increased to accommodate the image embedding
 )
 
 respoonse = llm.create_chat_completion(

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/README.md

@@ -516,7 +516,7 @@ chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin")
 llm = Llama(
   model_path="./path/to/llava/llama-model.gguf",
   chat_handler=chat_handler,
-  n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
+  n_ctx=2048, # n_ctx should be increased to accommodate the image embedding
 )
 llm.create_chat_completion(
     messages = [
@@ -547,7 +547,7 @@ llm = Llama.from_pretrained(
   repo_id="vikhyatk/moondream2",
   filename="*text-model*",
   chat_handler=chat_handler,
-  n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
+  n_ctx=2048, # n_ctx should be increased to accommodate the image embedding
 )
 
 respoonse = llm.create_chat_completion(

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.69"
+__version__ = "0.2.70"

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/llama_cpp/_internals.py

@@ -203,7 +203,7 @@ class _LlamaModel:
         # NOTE: Llama1 models automatically added a space at the start of the prompt
         # this line removes a leading space if the first token is a beginning of sentence token
         return (
-            output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() else output
+            output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b' ' else output
         )
 
     # Extra
@@ -812,4 +812,4 @@ class _LlamaSamplingContext:
     def accept(self, ctx_main: _LlamaContext, id: int, apply_grammar: bool):
         if apply_grammar and self.grammar is not None:
             ctx_main.grammar_accept_token(self.grammar, id)
-        self.prev.append(id)
+        self.prev.append(id)

{llama_cpp_python-0.2.69 → llama_cpp_python-0.2.70}/llama_cpp/llama.py

@@ -262,7 +262,12 @@ class Llama:
                         raise ValueError(f"Value for {k} is too long: {v}")
                     v_bytes = v_bytes.ljust(128, b"\0")
                     self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR
-                    self._kv_overrides_array[i].value.str_value[:128] = v_bytes
+                    # copy min(v_bytes, 128) to str_value
+                    ctypes.memmove(
+                        self._kv_overrides_array[i].value.str_value,
+                        v_bytes,
+                        min(len(v_bytes), 128),
+                    )
                 else:
                     raise ValueError(f"Unknown value type for {k}: {v}")
 
@@ -446,7 +451,7 @@ class Llama:
         if self.chat_format is None and self.chat_handler is None:
             self.chat_format = "llama-2"
             if self.verbose:
-                print(f"Using fallback chat format: {chat_format}", file=sys.stderr)
+                print(f"Using fallback chat format: {self.chat_format}", file=sys.stderr)
 
     @property
     def ctx(self) -> llama_cpp.llama_context_p:
@@ -950,18 +955,53 @@ class Llama:
 
         completion_id: str = f"cmpl-{str(uuid.uuid4())}"
         created: int = int(time.time())
+        prefix_token_id: int = int(self.metadata.get("tokenizer.ggml.prefix_token_id", self._model.token_prefix()))
+        middle_token_id: int = int(self.metadata.get("tokenizer.ggml.middle_token_id", self._model.token_middle()))
+        suffix_token_id: int = int(self.metadata.get("tokenizer.ggml.suffix_token_id", self._model.token_suffix()))
         # If prompt is empty, initialize completion with BOS token to avoid
         # detokenization including a space at the beginning of the completion
         completion_tokens: List[int] = [] if len(prompt) > 0 else [self.token_bos()]
         # Add blank space to start of prompt to match OG llama tokenizer
         prompt_tokens: List[int] = (
             (
-                self.tokenize(prompt.encode("utf-8"), special=True)
-                if prompt != ""
-                else [self.token_bos()]
+                [prefix_token_id]
+                if prefix_token_id >= 0 and suffix is not None
+                else []
+            )
+            +
+            (
+                (
+                    self.tokenize(prompt.encode("utf-8"), add_bos=(prefix_token_id < 0 or suffix is None), special=(prefix_token_id < 0 or suffix is None))
+                    if prompt != ""
+                    else (
+                        []
+                        if prefix_token_id >= 0 and suffix is not None
+                        else [self.token_bos()]
+                    )
+                )
+                if isinstance(prompt, str)
+                else prompt
+            )
+            +
+            (
+                (
+                    [suffix_token_id]
+                    +
+                    (
+                        self.tokenize(suffix.encode("utf-8"), add_bos=False, special=False)
+                        if suffix
+                        else []
+                    )
+                )
+                if suffix_token_id >= 0 and suffix is not None
+                else []
+            )
+            +
+            (
+                [middle_token_id]
+                if middle_token_id >= 0 and suffix is not None
+                else []
             )
-            if isinstance(prompt, str)
-            else prompt
         )
         text: bytes = b""
         returned_tokens: int = 0
@@ -1341,7 +1381,7 @@ class Llama:
         if echo:
             text_str = prompt + text_str
 
-        if suffix is not None:
+        if suffix_token_id < 0 and suffix is not None:
             text_str = text_str + suffix
 
         logprobs_or_none: Optional[CompletionLogprobs] = None