llama-cpp-python 0.2.59__tar.gz → 0.2.61__tar.gz

llama_cpp_python-0.2.61/.git/FETCH_HEAD

@@ -0,0 +1 @@
+ef29235d453ecf552f015f6ec04270b233bb22c9		'ef29235d453ecf552f015f6ec04270b233bb22c9' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.61/.git/HEAD

@@ -0,0 +1 @@
+ef29235d453ecf552f015f6ec04270b233bb22c9

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzRPWHJKQ0tPeTR3Y25uYWVBMmF3VE9IMFhtZzNOVTJCM3hYeQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzZIOVpZTXRtWE1KbUFkSkI4bXBLU2pKT3dMNTBFWjRPSTBxdQ==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.61/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 ef29235d453ecf552f015f6ec04270b233bb22c9 runner <runner@fv-az1215-545.(none)> 1712735396 +0000	checkout: moving from master to refs/tags/v0.2.61

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/FETCH_HEAD

@@ -0,0 +1 @@
+ba5e134e073ec6837078c874aba44a702944a676		'ba5e134e073ec6837078c874aba44a702944a676' of https://github.com/ggerganov/llama.cpp

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+ba5e134e073ec6837078c874aba44a702944a676

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzRPWHJKQ0tPeTR3Y25uYWVBMmF3VE9IMFhtZzNOVTJCM3hYeQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzZIOVpZTXRtWE1KbUFkSkI4bXBLU2pKT3dMNTBFWjRPSTBxdQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f runner <runner@fv-az1215-545.(none)> 1712735397 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f ba5e134e073ec6837078c874aba44a702944a676 runner <runner@fv-az1215-545.(none)> 1712735397 +0000	checkout: moving from master to ba5e134e073ec6837078c874aba44a702944a676

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f runner <runner@fv-az1215-545.(none)> 1712735397 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f runner <runner@fv-az1215-545.(none)> 1712735397 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzRPWHJKQ0tPeTR3Y25uYWVBMmF3VE9IMFhtZzNOVTJCM3hYeQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzXzZIOVpZTXRtWE1KbUFkSkI4bXBLU2pKT3dMNTBFWjRPSTBxdQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1215-545.(none)> 1712735398 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az1215-545.(none)> 1712735398 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1215-545.(none)> 1712735398 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1215-545.(none)> 1712735398 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f refs/remotes/origin/master

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f

llama_cpp_python-0.2.61/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1,2 @@
+67fac4b95fcccfda8ab965e9ba4992a9ddf3a25f
+ba5e134e073ec6837078c874aba44a702944a676

llama_cpp_python-0.2.61/.git/refs/tags/v0.2.61

@@ -0,0 +1 @@
+ef29235d453ecf552f015f6ec04270b233bb22c9

llama_cpp_python-0.2.61/.git/shallow

@@ -0,0 +1 @@
+ef29235d453ecf552f015f6ec04270b233bb22c9

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.61]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@ba5e134e073ec6837078c874aba44a702944a676
+- fix: pass correct type to chat handlers for chat completion logprobs by @abetlen in bb65b4d76411112c6fb0bf759efd746f99ef3c6b
+- feat: Add support for yaml based server configs by @abetlen in 060bfa64d529ade2af9b1f4e207a3937bbc4138f
+- feat: Add typechecking for ctypes structure attributes by @abetlen in 1347e1d050fc5a9a32ffe0bb3e22858da28003bd
+
+## [0.2.60]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@75cd4c77292034ecec587ecb401366f57338f7c0
+- fix: Always embed metal library by @abetlen in b3bfea6dbfb6ed9ce18f9a2723e0a9e4bd1da7ad
+- fix: missing logprobs in response, incorrect response type for functionary by @abetlen in 1ae3abbcc3af7f4a25a3ffc40b246f18039565e8
+- fix(docs): incorrect tool_choice example by @CISC in #1330
+
 ## [0.2.59]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/CMakeLists.txt

@@ -18,7 +18,7 @@ if (LLAMA_BUILD)
         set(LLAMA_F16C "Off" CACHE BOOL "llama: enable F16C" FORCE)
     endif()
 
-    if (APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
+    if (APPLE)
         set(LLAMA_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE)
     endif()
 

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.59
+Version: 0.2.61
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -24,6 +24,7 @@ Requires-Dist: fastapi>=0.100.0; extra == "server"
 Requires-Dist: pydantic-settings>=2.0.1; extra == "server"
 Requires-Dist: sse-starlette>=1.6.1; extra == "server"
 Requires-Dist: starlette-context<0.4,>=0.3.6; extra == "server"
+Requires-Dist: PyYAML>=5.1; extra == "server"
 Requires-Dist: pytest>=7.4.0; extra == "test"
 Requires-Dist: httpx>=0.24.1; extra == "test"
 Requires-Dist: scipy>=1.10; extra == "test"
@@ -87,6 +88,15 @@ This will also build `llama.cpp` from source and install it alongside this pytho
 
 If this fails, add `--verbose` to the `pip install` see the full cmake build log.
 
+**Pre-built Wheel (New)**
+
+It is also possible to install a pre-built wheel with basic CPU support.
+
+```bash
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+```
+
 ### Installation Configuration
 
 `llama.cpp` supports a number of hardware acceleration backends to speed up inference as well as backend specific options. See the [llama.cpp README](https://github.com/ggerganov/llama.cpp#build) for a full list.
@@ -143,12 +153,36 @@ CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-
 </details>
 
 <details>
-<summary>cuBLAS (CUDA)</summary>
+<summary>CUDA</summary>
+
+To install with CUDA support, set the `LLAMA_CUDA=on` environment variable before installing:
+
+```bash
+CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python
+```
+
+**Pre-built Wheel (New)**
+
+It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements:
+
+- CUDA Version is 12.1, 12.2 or 12.3
+- Python Version is 3.10, 3.11 or 3.12
 
-To install with cuBLAS, set the `LLAMA_CUBLAS=on` environment variable before installing:
+```bash
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/<cuda-version>
+```
+
+Where `<cuda-version>` is one of the following:
+- `cu121`: CUDA 12.1
+- `cu122`: CUDA 12.2
+- `cu123`: CUDA 12.3
+
+For example, to install the CUDA 12.1 wheel:
 
 ```bash
-CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
 ```
 
 </details>
@@ -162,6 +196,18 @@ To install with Metal (MPS), set the `LLAMA_METAL=on` environment variable befor
 CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python
 ```
 
+**Pre-built Wheel (New)**
+
+It is also possible to install a pre-built wheel with Metal support. As long as your system meets some requirements:
+
+- MacOS Version is 11.0 or later
+- Python Version is 3.10, 3.11 or 3.12
+
+```bash
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal
+```
+
 </details>
 <details>
 
@@ -456,12 +502,12 @@ The high-level API supports OpenAI compatible function and tool calling. This is
           }
         }
       }],
-      tool_choice=[{
+      tool_choice={
         "type": "function",
         "function": {
           "name": "UserDetail"
         }
-      }]
+      }
 )
 ```
 
@@ -612,7 +658,7 @@ python3 -m llama_cpp.server --model models/7B/llama-model.gguf
 Similar to Hardware Acceleration section above, you can also install with GPU (cuBLAS) support like this:
 
 ```bash
-CMAKE_ARGS="-DLLAMA_CUBLAS=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]'
+CMAKE_ARGS="-DLLAMA_CUDA=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]'
 python3 -m llama_cpp.server --model models/7B/llama-model.gguf --n_gpu_layers 35
 ```
 

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/README.md

@@ -44,6 +44,15 @@ This will also build `llama.cpp` from source and install it alongside this pytho
 
 If this fails, add `--verbose` to the `pip install` see the full cmake build log.
 
+**Pre-built Wheel (New)**
+
+It is also possible to install a pre-built wheel with basic CPU support.
+
+```bash
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+```
+
 ### Installation Configuration
 
 `llama.cpp` supports a number of hardware acceleration backends to speed up inference as well as backend specific options. See the [llama.cpp README](https://github.com/ggerganov/llama.cpp#build) for a full list.
@@ -100,12 +109,36 @@ CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-
 </details>
 
 <details>
-<summary>cuBLAS (CUDA)</summary>
+<summary>CUDA</summary>
+
+To install with CUDA support, set the `LLAMA_CUDA=on` environment variable before installing:
+
+```bash
+CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python
+```
+
+**Pre-built Wheel (New)**
+
+It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements:
+
+- CUDA Version is 12.1, 12.2 or 12.3
+- Python Version is 3.10, 3.11 or 3.12
 
-To install with cuBLAS, set the `LLAMA_CUBLAS=on` environment variable before installing:
+```bash
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/<cuda-version>
+```
+
+Where `<cuda-version>` is one of the following:
+- `cu121`: CUDA 12.1
+- `cu122`: CUDA 12.2
+- `cu123`: CUDA 12.3
+
+For example, to install the CUDA 12.1 wheel:
 
 ```bash
-CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
 ```
 
 </details>
@@ -119,6 +152,18 @@ To install with Metal (MPS), set the `LLAMA_METAL=on` environment variable befor
 CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python
 ```
 
+**Pre-built Wheel (New)**
+
+It is also possible to install a pre-built wheel with Metal support. As long as your system meets some requirements:
+
+- MacOS Version is 11.0 or later
+- Python Version is 3.10, 3.11 or 3.12
+
+```bash
+pip install llama-cpp-python \
+  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal
+```
+
 </details>
 <details>
 
@@ -413,12 +458,12 @@ The high-level API supports OpenAI compatible function and tool calling. This is
           }
         }
       }],
-      tool_choice=[{
+      tool_choice={
         "type": "function",
         "function": {
           "name": "UserDetail"
         }
-      }]
+      }
 )
 ```
 
@@ -569,7 +614,7 @@ python3 -m llama_cpp.server --model models/7B/llama-model.gguf
 Similar to Hardware Acceleration section above, you can also install with GPU (cuBLAS) support like this:
 
 ```bash
-CMAKE_ARGS="-DLLAMA_CUBLAS=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]'
+CMAKE_ARGS="-DLLAMA_CUDA=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]'
 python3 -m llama_cpp.server --model models/7B/llama-model.gguf --n_gpu_layers 35
 ```
 

llama_cpp_python-0.2.61/examples/batch-processing/server.py

@@ -0,0 +1,30 @@
+"""llama-cpp-python server from scratch in a single file.
+"""
+
+# import llama_cpp
+
+# path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf"
+
+# model_params = llama_cpp.llama_model_default_params()
+# model = llama_cpp.llama_load_model_from_file(path, model_params)
+
+# if model is None:
+#     raise RuntimeError(f"Failed to load model from file: {path}")
+
+
+# ctx_params = llama_cpp.llama_context_default_params()
+# ctx = llama_cpp.llama_new_context_with_model(model, ctx_params)
+
+# if ctx is None:
+#     raise RuntimeError("Failed to create context")
+
+
+from fastapi import FastAPI
+
+app = FastAPI()
+
+import openai.types.chat as types
+
+@app.post("/v1/chat/completions")
+def create_chat_completions():
+    return {"message": "Hello World"}

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.59"
+__version__ = "0.2.61"

{llama_cpp_python-0.2.59 → llama_cpp_python-0.2.61}/llama_cpp/llama.py

@@ -1664,7 +1664,8 @@ class Llama:
             top_k=top_k,
             min_p=min_p,
             typical_p=typical_p,
-            logprobs=top_logprobs if logprobs else None,
+            logprobs=logprobs,
+            top_logprobs=top_logprobs,
             stream=stream,
             stop=stop,
             seed=seed,