llama-cpp-python 0.2.61__tar.gz → 0.2.63__tar.gz

llama_cpp_python-0.2.63/.git/FETCH_HEAD

@@ -0,0 +1 @@
+02812148635bf6337ffc7d1abb34093f4065df88		'02812148635bf6337ffc7d1abb34093f4065df88' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.63/.git/HEAD

@@ -0,0 +1 @@
+02812148635bf6337ffc7d1abb34093f4065df88

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzZIOVpZTXRtWE1KbUFkSkI4bXBLU2pKT3dMNTBFWjRPSTBxdQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3pWRkFjdDdJMG5wSmI5RlY5WHFMUTAzUkl6UXFqbTN1WGJmRA==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.63/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 02812148635bf6337ffc7d1abb34093f4065df88 runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586238 +0000	checkout: moving from master to refs/tags/v0.2.63

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+0e4802b2ecbaab04b4f829fde4a3096ca19c84b5

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzZIOVpZTXRtWE1KbUFkSkI4bXBLU2pKT3dMNTBFWjRPSTBxdQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3pWRkFjdDdJMG5wSmI5RlY5WHFMUTAzUkl6UXFqbTN1WGJmRA==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586239 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586239 +0000	checkout: moving from master to 0e4802b2ecbaab04b4f829fde4a3096ca19c84b5

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586239 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586239 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzZIOVpZTXRtWE1KbUFkSkI4bXBLU2pKT3dMNTBFWjRPSTBxdQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3pWRkFjdDdJMG5wSmI5RlY5WHFMUTAzUkl6UXFqbTN1WGJmRA==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586240 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586241 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586240 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az840-923.wndg40qfsgie5kikvjficeueua.ex.internal.cloudapp.net> 1713586240 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 refs/remotes/origin/master

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+0e4802b2ecbaab04b4f829fde4a3096ca19c84b5

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/refs/tags/b2699

@@ -0,0 +1 @@
+0e4802b2ecbaab04b4f829fde4a3096ca19c84b5

llama_cpp_python-0.2.63/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1 @@
+0e4802b2ecbaab04b4f829fde4a3096ca19c84b5

llama_cpp_python-0.2.63/.git/refs/tags/v0.2.63

@@ -0,0 +1 @@
+02812148635bf6337ffc7d1abb34093f4065df88

llama_cpp_python-0.2.63/.git/shallow

@@ -0,0 +1 @@
+02812148635bf6337ffc7d1abb34093f4065df88

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.63]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@0e4802b2ecbaab04b4f829fde4a3096ca19c84b5
+- feat: Add stopping_criteria to ChatFormatter, allow stopping on arbitrary token ids, fixes llama3 instruct by @abetlen in cc81afebf04d26ca1ac3cf72f23f18da6ab58588
+
+## [0.2.62]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@3b8f1ec4b18770531d0b1d792f3edf08254e4f0c
+- feat: update grammar schema converter to match llama.cpp by @themrzmaster in #1353
+- feat: add disable_ping_events flag by @khimaros in #1257
+- feat: Make saved state more compact on-disk by @tc-wolf in #1296
+- feat: Use all available CPUs for batch processing by @ddh0 in #1345
+
 ## [0.2.61]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@ba5e134e073ec6837078c874aba44a702944a676

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.61
+Version: 0.2.63
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.61"
+__version__ = "0.2.63"

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/llama_cpp/llama.py

@@ -18,6 +18,7 @@ from typing import (
     Iterator,
     Deque,
     Callable,
+    Dict,
 )
 from collections import deque
 from pathlib import Path
@@ -262,9 +263,7 @@ class Llama:
 
         self.n_batch = min(n_ctx, n_batch)  # ???
         self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1)
-        self.n_threads_batch = n_threads_batch or max(
-            multiprocessing.cpu_count() // 2, 1
-        )
+        self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count()
 
         # Context Params
         self.context_params = llama_cpp.llama_context_default_params()
@@ -427,7 +426,10 @@ class Llama:
                     print(f"Using chat bos_token: {bos_token}", file=sys.stderr)
 
                 self.chat_handler = llama_chat_format.Jinja2ChatFormatter(
-                    template=template, eos_token=eos_token, bos_token=bos_token
+                    template=template,
+                    eos_token=eos_token,
+                    bos_token=bos_token,
+                    stop_token_ids=[eos_token_id],
                 ).to_chat_handler()
 
         if self.chat_format is None and self.chat_handler is None:
@@ -1793,7 +1795,7 @@ class Llama:
                 file=sys.stderr,
             )
         return LlamaState(
-            scores=self.scores.copy(),
+            scores=self._scores.copy(),
             input_ids=self.input_ids.copy(),
             n_tokens=self.n_tokens,
             llama_state=bytes(llama_state_compact),
@@ -1802,7 +1804,9 @@ class Llama:
 
     def load_state(self, state: LlamaState) -> None:
         assert self._ctx.ctx is not None
-        self.scores = state.scores.copy()
+        # Only filling in up to `n_tokens` and then zero-ing out the rest
+        self.scores[: state.n_tokens, :] = state.scores.copy()
+        self.scores[state.n_tokens :, :] = 0.0
         self.input_ids = state.input_ids.copy()
         self.n_tokens = state.n_tokens
         state_size = state.llama_state_size
@@ -1953,7 +1957,6 @@ class Llama:
                 local_dir_use_symlinks=local_dir_use_symlinks,
                 cache_dir=cache_dir,
                 local_files_only=True,
-
             )
         else:
             model_path = os.path.join(local_dir, filename)

{llama_cpp_python-0.2.61 → llama_cpp_python-0.2.63}/llama_cpp/llama_chat_format.py

@@ -10,6 +10,9 @@ from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, P
 
 import jinja2
 
+import numpy as np
+import numpy.typing as npt
+
 import llama_cpp.llama as llama
 import llama_cpp.llama_types as llama_types
 import llama_cpp.llama_grammar as llama_grammar
@@ -150,6 +153,7 @@ class ChatFormatterResponse:
 
     prompt: str
     stop: Optional[Union[str, List[str]]] = None
+    stopping_criteria: Optional[llama.StoppingCriteriaList] = None
 
 
 class ChatFormatter(Protocol):
@@ -173,12 +177,14 @@ class Jinja2ChatFormatter(ChatFormatter):
         eos_token: str,
         bos_token: str,
         add_generation_prompt: bool = True,
+        stop_token_ids: Optional[List[int]] = None,
     ):
         """A chat formatter that uses jinja2 templates to format the prompt."""
         self.template = template
         self.eos_token = eos_token
         self.bos_token = bos_token
         self.add_generation_prompt = add_generation_prompt
+        self.stop_token_ids = set(stop_token_ids) if stop_token_ids is not None else None
 
         self._environment = jinja2.Environment(
             loader=jinja2.BaseLoader(),
@@ -211,7 +217,16 @@ class Jinja2ChatFormatter(ChatFormatter):
             tool_choice=tool_choice,
         )
 
-        return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token])
+        stopping_criteria = None
+        if self.stop_token_ids is not None:
+            def stop_on_last_token(
+                    tokens: npt.NDArray[np.intc],
+                    logits: npt.NDArray[np.single]
+            ) -> bool:
+                return tokens[-1] in self.stop_token_ids
+            stopping_criteria = llama.StoppingCriteriaList([stop_on_last_token])
+
+        return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token], stopping_criteria=stopping_criteria)
 
     def to_chat_handler(self) -> LlamaChatCompletionHandler:
         return chat_formatter_to_chat_completion_handler(self)
@@ -533,6 +548,10 @@ def chat_formatter_to_chat_completion_handler(
             rstop = result.stop if isinstance(result.stop, list) else [result.stop]
             stop = stop + rstop
 
+        stopping_criteria = None
+        if result.stopping_criteria is not None:
+            stopping_criteria = result.stopping_criteria
+
         if response_format is not None and response_format["type"] == "json_object":
             grammar = _grammar_for_response_format(response_format, verbose=llama.verbose)
 
@@ -598,6 +617,7 @@ def chat_formatter_to_chat_completion_handler(
             mirostat_eta=mirostat_eta,
             model=model,
             logits_processor=logits_processor,
+            stopping_criteria=stopping_criteria,
             grammar=grammar,
             logit_bias=logit_bias,
         )
@@ -2709,4 +2729,4 @@ def chatml_function_calling(
             },
         }
 
-    raise ValueError("Automatic streaming tool choice is not supported")
+    raise ValueError("Automatic streaming tool choice is not supported")