llama-cpp-python 0.2.60__tar.gz → 0.2.62__tar.gz

llama_cpp_python-0.2.62/.git/FETCH_HEAD

@@ -0,0 +1 @@
+893a27a7364366cc195dcc50aadfa75d95bb9319		'893a27a7364366cc195dcc50aadfa75d95bb9319' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.62/.git/HEAD

@@ -0,0 +1 @@
+893a27a7364366cc195dcc50aadfa75d95bb9319

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzVMZFpGUjhRNkxlRTRIUkZSUlg1cjM0QU1zY2hzcTBjTDg3Nw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX2g4ejgzMTE4dlV0N3pXMXBJa3ZtUkxLSGtnZERNaTFqTzZuag==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.62/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 893a27a7364366cc195dcc50aadfa75d95bb9319 runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419074 +0000	checkout: moving from master to refs/tags/v0.2.62

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+3b8f1ec4b18770531d0b1d792f3edf08254e4f0c

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzVMZFpGUjhRNkxlRTRIUkZSUlg1cjM0QU1zY2hzcTBjTDg3Nw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX2g4ejgzMTE4dlV0N3pXMXBJa3ZtUkxLSGtnZERNaTFqTzZuag==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419075 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+3b8f1ec4b18770531d0b1d792f3edf08254e4f0c 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419075 +0000	checkout: moving from master to 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419075 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419075 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzVMZFpGUjhRNkxlRTRIUkZSUlg1cjM0QU1zY2hzcTBjTDg3Nw==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX2g4ejgzMTE4dlV0N3pXMXBJa3ZtUkxLSGtnZERNaTFqTzZuag==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419076 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419077 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419076 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az585-611.vitsawovrx5upos552vgumbwsa.phxx.internal.cloudapp.net> 1713419076 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+3b8f1ec4b18770531d0b1d792f3edf08254e4f0c refs/remotes/origin/master

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+3b8f1ec4b18770531d0b1d792f3edf08254e4f0c

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/refs/tags/b2690

@@ -0,0 +1 @@
+3b8f1ec4b18770531d0b1d792f3edf08254e4f0c

llama_cpp_python-0.2.62/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1 @@
+3b8f1ec4b18770531d0b1d792f3edf08254e4f0c

llama_cpp_python-0.2.62/.git/refs/tags/v0.2.62

@@ -0,0 +1 @@
+893a27a7364366cc195dcc50aadfa75d95bb9319

llama_cpp_python-0.2.62/.git/shallow

@@ -0,0 +1 @@
+893a27a7364366cc195dcc50aadfa75d95bb9319

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.62]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@3b8f1ec4b18770531d0b1d792f3edf08254e4f0c
+- feat: update grammar schema converter to match llama.cpp by @themrzmaster in #1353
+- feat: add disable_ping_events flag by @khimaros in #1257
+- feat: Make saved state more compact on-disk by @tc-wolf in #1296
+- feat: Use all available CPUs for batch processing by @ddh0 in #1345
+
+## [0.2.61]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@ba5e134e073ec6837078c874aba44a702944a676
+- fix: pass correct type to chat handlers for chat completion logprobs by @abetlen in bb65b4d76411112c6fb0bf759efd746f99ef3c6b
+- feat: Add support for yaml based server configs by @abetlen in 060bfa64d529ade2af9b1f4e207a3937bbc4138f
+- feat: Add typechecking for ctypes structure attributes by @abetlen in 1347e1d050fc5a9a32ffe0bb3e22858da28003bd
+
 ## [0.2.60]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@75cd4c77292034ecec587ecb401366f57338f7c0

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.60
+Version: 0.2.62
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -24,6 +24,7 @@ Requires-Dist: fastapi>=0.100.0; extra == "server"
 Requires-Dist: pydantic-settings>=2.0.1; extra == "server"
 Requires-Dist: sse-starlette>=1.6.1; extra == "server"
 Requires-Dist: starlette-context<0.4,>=0.3.6; extra == "server"
+Requires-Dist: PyYAML>=5.1; extra == "server"
 Requires-Dist: pytest>=7.4.0; extra == "test"
 Requires-Dist: httpx>=0.24.1; extra == "test"
 Requires-Dist: scipy>=1.10; extra == "test"

llama_cpp_python-0.2.62/examples/batch-processing/server.py

@@ -0,0 +1,30 @@
+"""llama-cpp-python server from scratch in a single file.
+"""
+
+# import llama_cpp
+
+# path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf"
+
+# model_params = llama_cpp.llama_model_default_params()
+# model = llama_cpp.llama_load_model_from_file(path, model_params)
+
+# if model is None:
+#     raise RuntimeError(f"Failed to load model from file: {path}")
+
+
+# ctx_params = llama_cpp.llama_context_default_params()
+# ctx = llama_cpp.llama_new_context_with_model(model, ctx_params)
+
+# if ctx is None:
+#     raise RuntimeError("Failed to create context")
+
+
+from fastapi import FastAPI
+
+app = FastAPI()
+
+import openai.types.chat as types
+
+@app.post("/v1/chat/completions")
+def create_chat_completions():
+    return {"message": "Hello World"}

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.60"
+__version__ = "0.2.62"

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/llama_cpp/llama.py

@@ -18,6 +18,7 @@ from typing import (
     Iterator,
     Deque,
     Callable,
+    Dict,
 )
 from collections import deque
 from pathlib import Path
@@ -262,9 +263,7 @@ class Llama:
 
         self.n_batch = min(n_ctx, n_batch)  # ???
         self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1)
-        self.n_threads_batch = n_threads_batch or max(
-            multiprocessing.cpu_count() // 2, 1
-        )
+        self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count()
 
         # Context Params
         self.context_params = llama_cpp.llama_context_default_params()
@@ -1664,7 +1663,8 @@ class Llama:
             top_k=top_k,
             min_p=min_p,
             typical_p=typical_p,
-            logprobs=top_logprobs if logprobs else None,
+            logprobs=logprobs,
+            top_logprobs=top_logprobs,
             stream=stream,
             stop=stop,
             seed=seed,
@@ -1792,7 +1792,7 @@ class Llama:
                 file=sys.stderr,
             )
         return LlamaState(
-            scores=self.scores.copy(),
+            scores=self._scores.copy(),
             input_ids=self.input_ids.copy(),
             n_tokens=self.n_tokens,
             llama_state=bytes(llama_state_compact),
@@ -1801,7 +1801,9 @@ class Llama:
 
     def load_state(self, state: LlamaState) -> None:
         assert self._ctx.ctx is not None
-        self.scores = state.scores.copy()
+        # Only filling in up to `n_tokens` and then zero-ing out the rest
+        self.scores[: state.n_tokens, :] = state.scores.copy()
+        self.scores[state.n_tokens :, :] = 0.0
         self.input_ids = state.input_ids.copy()
         self.n_tokens = state.n_tokens
         state_size = state.llama_state_size
@@ -1952,7 +1954,6 @@ class Llama:
                 local_dir_use_symlinks=local_dir_use_symlinks,
                 cache_dir=cache_dir,
                 local_files_only=True,
-
             )
         else:
             model_path = os.path.join(local_dir, filename)

{llama_cpp_python-0.2.60 → llama_cpp_python-0.2.62}/llama_cpp/llama_chat_format.py

@@ -77,6 +77,8 @@ class LlamaChatCompletionHandler(Protocol):
         mirostat_eta: float = 0.1,
         logits_processor: Optional[llama.LogitsProcessorList] = None,
         grammar: Optional[llama.LlamaGrammar] = None,
+        logprobs: Optional[bool] = None,
+        top_logprobs: Optional[int] = None,
         **kwargs,  # type: ignore
     ) -> Union[
         llama_types.CreateChatCompletionResponse,
@@ -338,7 +340,7 @@ def _convert_completion_to_chat_function(
                             }
                         ],
                     },
-                    "logprobs": None,
+                    "logprobs": completion["choices"][0]["logprobs"],
                     "finish_reason": "tool_calls",
                 }
             ],
@@ -391,7 +393,7 @@ def _convert_completion_to_chat_function(
                             {
                                 "index": 0,
                                 "finish_reason": None,
-                                "logprobs": None,
+                                "logprobs": chunk["choices"][0]["logprobs"],
                                 "delta": {
                                     "role": None,
                                     "content": None,
@@ -426,7 +428,7 @@ def _convert_completion_to_chat_function(
                         {
                             "index": 0,
                             "finish_reason": None,
-                            "logprobs": None,
+                            "logprobs": chunk["choices"][0]["logprobs"],
                             "delta": {
                                 "role": None,
                                 "content": None,
@@ -491,7 +493,6 @@ def chat_formatter_to_chat_completion_handler(
         temperature: float = 0.2,
         top_p: float = 0.95,
         top_k: int = 40,
-        logprobs: int = 0,
         min_p: float = 0.05,
         typical_p: float = 1.0,
         stream: bool = False,
@@ -512,6 +513,8 @@ def chat_formatter_to_chat_completion_handler(
         logits_processor: Optional[llama.LogitsProcessorList] = None,
         grammar: Optional[llama.LlamaGrammar] = None,
         logit_bias: Optional[Dict[str, float]] = None,
+        logprobs: Optional[bool] = None,
+        top_logprobs: Optional[int] = None,
         **kwargs,  # type: ignore
     ) -> Union[
         llama_types.CreateChatCompletionResponse,
@@ -581,7 +584,7 @@ def chat_formatter_to_chat_completion_handler(
             top_k=top_k,
             min_p=min_p,
             typical_p=typical_p,
-            logprobs=logprobs,
+            logprobs=top_logprobs if logprobs else None,
             stream=stream,
             stop=stop,
             seed=seed,
@@ -1628,7 +1631,7 @@ def functionary_chat_handler(
                         }
                     ],
                 },
-                "logprobs": None,
+                "logprobs": completion["choices"][0]["logprobs"],
                 "finish_reason": "tool_calls",
             }
         ],
@@ -2085,7 +2088,7 @@ def functionary_v1_v2_chat_handler(
         choices=[
             {
                 "index": 0,
-                "logprobs": None,
+                "logprobs": completion["choices"][0]["logprobs"],
                 "message": {
                     "role": "assistant",
                     "content": None if content == "" else content,
@@ -2311,11 +2314,14 @@ def chatml_function_calling(
     model: Optional[str] = None,
     logits_processor: Optional[llama.LogitsProcessorList] = None,
     grammar: Optional[llama.LlamaGrammar] = None,
+    logprobs: Optional[bool] = None,
+    top_logprobs: Optional[int] = None,
     **kwargs,  # type: ignore
 ) -> Union[
     llama_types.CreateChatCompletionResponse,
     Iterator[llama_types.CreateChatCompletionStreamResponse],
 ]:
+    print(logprobs)
     function_calling_template = (
         "{% for message in messages %}"
         "<|im_start|>{{ message.role }}\n"
@@ -2437,6 +2443,7 @@ def chatml_function_calling(
                 model=model,
                 logits_processor=logits_processor,
                 grammar=grammar,
+                logprobs=top_logprobs if logprobs else None,
             ),
             stream=stream,
         )
@@ -2549,6 +2556,7 @@ def chatml_function_calling(
                 typical_p=typical_p,
                 stream=stream,
                 stop=["<|im_end|>"],
+                logprobs=top_logprobs if logprobs else None,
                 max_tokens=None,
                 presence_penalty=presence_penalty,
                 frequency_penalty=frequency_penalty,
@@ -2660,7 +2668,7 @@ def chatml_function_calling(
                 {
                     "finish_reason": "tool_calls",
                     "index": 0,
-                    "logprobs": None,
+                    "logprobs": completion["choices"][0]["logprobs"],
                     "message": {
                         "role": "assistant",
                         "content": None,
@@ -2701,4 +2709,4 @@ def chatml_function_calling(
             },
         }
 
-    raise ValueError("Automatic streaming tool choice is not supported")
+    raise ValueError("Automatic streaming tool choice is not supported")