llama-cpp-python 0.2.67__tar.gz → 0.2.69__tar.gz

llama_cpp_python-0.2.69/.git/FETCH_HEAD

@@ -0,0 +1 @@
+21171223967896b9e941007841ebd1513b3cc4b9		'21171223967896b9e941007841ebd1513b3cc4b9' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.69/.git/HEAD

@@ -0,0 +1 @@
+21171223967896b9e941007841ebd1513b3cc4b9

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1Z3cnF2cFBtMHRHQmp6b0tWNHNLd1ZOYXVFdDYyYzF3amxueQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3k2Y2NrSEplV2ZpcDZGZnR6VldrRzlDQVBhQUdIYTNaOTJxNQ==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.69/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 21171223967896b9e941007841ebd1513b3cc4b9 runner <runner@fv-az1055-577.(none)> 1714666134 +0000	checkout: moving from master to refs/tags/v0.2.69

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+6ecf3189e00a1e8e737a78b6d10e1d7006e050a2

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1Z3cnF2cFBtMHRHQmp6b0tWNHNLd1ZOYXVFdDYyYzF3amxueQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3k2Y2NrSEplV2ZpcDZGZnR6VldrRzlDQVBhQUdIYTNaOTJxNQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 runner <runner@fv-az1055-577.(none)> 1714666135 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 runner <runner@fv-az1055-577.(none)> 1714666135 +0000	checkout: moving from master to 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 runner <runner@fv-az1055-577.(none)> 1714666135 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 runner <runner@fv-az1055-577.(none)> 1714666135 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX1Z3cnF2cFBtMHRHQmp6b0tWNHNLd1ZOYXVFdDYyYzF3amxueQ==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3k2Y2NrSEplV2ZpcDZGZnR6VldrRzlDQVBhQUdIYTNaOTJxNQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1055-577.(none)> 1714666137 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az1055-577.(none)> 1714666138 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1055-577.(none)> 1714666137 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1055-577.(none)> 1714666137 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 refs/remotes/origin/master

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+6ecf3189e00a1e8e737a78b6d10e1d7006e050a2

llama_cpp_python-0.2.69/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1 @@
+6ecf3189e00a1e8e737a78b6d10e1d7006e050a2

llama_cpp_python-0.2.69/.git/refs/tags/v0.2.69

@@ -0,0 +1 @@
+21171223967896b9e941007841ebd1513b3cc4b9

llama_cpp_python-0.2.69/.git/shallow

@@ -0,0 +1 @@
+21171223967896b9e941007841ebd1513b3cc4b9

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/.github/workflows/build-and-release.yaml

@@ -39,7 +39,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: wheels
+          name: wheels-${{ matrix.os }}
           path: ./wheelhouse/*.whl
 
   build_wheels_arm64:
@@ -100,14 +100,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: release
-
-      - uses: actions/download-artifact@v3
-        with:
-          name: release
+          merge-multiple: true
           path: dist
 
       - uses: softprops/action-gh-release@v2

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.69]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@6ecf3189e00a1e8e737a78b6d10e1d7006e050a2
+- feat: Add llama-3-vision-alpha chat format by @abetlen in 31b1d95a6c19f5b615a3286069f181a415f872e8
+- fix: Change default verbose value of verbose in image chat format handlers to True to match Llama by @abetlen in 4f01c452b6c738dc56eacac3758119b12c57ea94
+- fix: Suppress all logs when verbose=False, use hardcoded fileno's to work in colab notebooks by @abetlen in f116175a5a7c84569c88cad231855c1e6e59ff6e
+- fix: UTF-8 handling with grammars by @jsoma in #1415
+
+## [0.2.68]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@77e15bec6217a39be59b9cc83d6b9afb6b0d8167
+- feat: Add option to enable flash_attn to Lllama params and ModelSettings by @abetlen in 22d77eefd2edaf0148f53374d0cac74d0e25d06e
+- fix(ci): Fix build-and-release.yaml by @Smartappli in #1413
+
 ## [0.2.67]
 
 - fix: Ensure image renders before text in chat formats regardless of message content order by @abetlen in 3489ef09d3775f4a87fb7114f619e8ba9cb6b656

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.67
+Version: 0.2.69
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -165,7 +165,7 @@ CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python
 
 It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements:
 
-- CUDA Version is 12.1, 12.2 or 12.3
+- CUDA Version is 12.1, 12.2, 12.3, or 12.4
 - Python Version is 3.10, 3.11 or 3.12
 
 ```bash
@@ -177,6 +177,7 @@ Where `<cuda-version>` is one of the following:
 - `cu121`: CUDA 12.1
 - `cu122`: CUDA 12.2
 - `cu123`: CUDA 12.3
+- `cu124`: CUDA 12.4
 
 For example, to install the CUDA 12.1 wheel:
 
@@ -320,20 +321,26 @@ The high-level API provides a simple managed interface through the [`Llama`](htt
 Below is a short example demonstrating how to use the high-level API to for basic text completion:
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(
+from llama_cpp import Llama
+
+llm = Llama(
       model_path="./models/7B/llama-model.gguf",
       # n_gpu_layers=-1, # Uncomment to use GPU acceleration
       # seed=1337, # Uncomment to set a specific seed
       # n_ctx=2048, # Uncomment to increase the context window
 )
->>> output = llm(
+output = llm(
       "Q: Name the planets in the solar system? A: ", # Prompt
       max_tokens=32, # Generate up to 32 tokens, set to None to generate up to the end of the context window
       stop=["Q:", "\n"], # Stop generating just before the model would generate a new question
       echo=True # Echo the prompt back in the output
 ) # Generate a completion, can also call create_completion
->>> print(output)
+print(output)
+```
+
+By default `llama-cpp-python` generates completions in an OpenAI compatible format:
+
+```python
 {
   "id": "cmpl-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
   "object": "text_completion",
@@ -388,12 +395,12 @@ The model will will format the messages into a single prompt using the following
 Set `verbose=True` to see the selected chat format.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(
+from llama_cpp import Llama
+llm = Llama(
       model_path="path/to/llama-2/llama-model.gguf",
       chat_format="llama-2"
 )
->>> llm.create_chat_completion(
+llm.create_chat_completion(
       messages = [
           {"role": "system", "content": "You are an assistant who perfectly describes images."},
           {
@@ -418,9 +425,9 @@ To constrain chat responses to only valid JSON or a specific JSON Schema use the
 The following example will constrain the response to valid JSON strings only.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
->>> llm.create_chat_completion(
+from llama_cpp import Llama
+llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
+llm.create_chat_completion(
     messages=[
         {
             "role": "system",
@@ -440,9 +447,9 @@ The following example will constrain the response to valid JSON strings only.
 To constrain the response further to a specific JSON Schema add the schema to the `schema` property of the `response_format` argument.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
->>> llm.create_chat_completion(
+from llama_cpp import Llama
+llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
+llm.create_chat_completion(
     messages=[
         {
             "role": "system",
@@ -467,9 +474,9 @@ To constrain the response further to a specific JSON Schema add the schema to th
 The high-level API supports OpenAI compatible function and tool calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling")
->>> llm.create_chat_completion(
+from llama_cpp import Llama
+llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling")
+llm.create_chat_completion(
       messages = [
         {
           "role": "system",
@@ -519,9 +526,9 @@ The various gguf-converted files for this set of models can be found [here](http
 Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is required to provide HF Tokenizer for functionary. The `LlamaHFTokenizer` class can be initialized and passed into the Llama class. This will override the default llama.cpp tokenizer used in Llama class. The tokenizer files are already included in the respective HF repositories hosting the gguf files.
 
 ```python
->>> from llama_cpp import Llama
->>> from llama_cpp.llama_tokenizer import LlamaHFTokenizer
->>> llm = Llama.from_pretrained(
+from llama_cpp import Llama
+from llama_cpp.llama_tokenizer import LlamaHFTokenizer
+llm = Llama.from_pretrained(
   repo_id="meetkai/functionary-small-v2.2-GGUF",
   filename="functionary-small-v2.2.q4_0.gguf",
   chat_format="functionary-v2",
@@ -547,15 +554,15 @@ You'll first need to download one of the available multi-modal models in GGUF fo
 Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images.
 
 ```python
->>> from llama_cpp import Llama
->>> from llama_cpp.llama_chat_format import Llava15ChatHandler
->>> chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin")
->>> llm = Llama(
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import Llava15ChatHandler
+chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin")
+llm = Llama(
   model_path="./path/to/llava/llama-model.gguf",
   chat_handler=chat_handler,
   n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
 )
->>> llm.create_chat_completion(
+llm.create_chat_completion(
     messages = [
         {"role": "system", "content": "You are an assistant who perfectly describes images."},
         {
@@ -572,19 +579,22 @@ Then you'll need to use a custom chat handler to load the clip model and process
 You can also pull the model from the Hugging Face Hub using the `from_pretrained` method.
 
 ```python
->>> from llama_cpp import Llama
->>> from llama_cpp.llama_chat_format import MoondreamChatHandler
->>> chat_handler = MoondreamChatHandler.from_pretrained(
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import MoondreamChatHandler
+
+chat_handler = MoondreamChatHandler.from_pretrained(
   repo_id="vikhyatk/moondream2",
   filename="*mmproj*",
 )
->>> llm = Llama.from_pretrained(
-  repo_id="vikhyatk/moondream2"
+
+llm = Llama.from_pretrained(
+  repo_id="vikhyatk/moondream2",
   filename="*text-model*",
   chat_handler=chat_handler,
   n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
 )
->>> llm.create_chat_completion(
+
+respoonse = llm.create_chat_completion(
     messages = [
         {
             "role": "user",
@@ -596,6 +606,7 @@ You can also pull the model from the Hugging Face Hub using the `from_pretrained
         }
     ]
 )
+print(response["choices"][0]["text"])
 ```
 
 **Note**: Multi-modal models also support tool calling and JSON mode.
@@ -748,18 +759,18 @@ The entire low-level API can be found in [llama_cpp/llama_cpp.py](https://github
 Below is a short example demonstrating how to use the low-level API to tokenize a prompt:
 
 ```python
->>> import llama_cpp
->>> import ctypes
->>> llama_cpp.llama_backend_init(False) # Must be called once at the start of each program
->>> params = llama_cpp.llama_context_default_params()
+import llama_cpp
+import ctypes
+llama_cpp.llama_backend_init(False) # Must be called once at the start of each program
+params = llama_cpp.llama_context_default_params()
 # use bytes for char * params
->>> model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params)
->>> ctx = llama_cpp.llama_new_context_with_model(model, params)
->>> max_tokens = params.n_ctx
+model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params)
+ctx = llama_cpp.llama_new_context_with_model(model, params)
+max_tokens = params.n_ctx
 # use ctypes arrays for array params
->>> tokens = (llama_cpp.llama_token * int(max_tokens))()
->>> n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True))
->>> llama_cpp.llama_free(ctx)
+tokens = (llama_cpp.llama_token * int(max_tokens))()
+n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True))
+llama_cpp.llama_free(ctx)
 ```
 
 Check out the [examples folder](examples/low_level_api) for more examples of using the low-level API.

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/README.md

@@ -121,7 +121,7 @@ CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python
 
 It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements:
 
-- CUDA Version is 12.1, 12.2 or 12.3
+- CUDA Version is 12.1, 12.2, 12.3, or 12.4
 - Python Version is 3.10, 3.11 or 3.12
 
 ```bash
@@ -133,6 +133,7 @@ Where `<cuda-version>` is one of the following:
 - `cu121`: CUDA 12.1
 - `cu122`: CUDA 12.2
 - `cu123`: CUDA 12.3
+- `cu124`: CUDA 12.4
 
 For example, to install the CUDA 12.1 wheel:
 
@@ -276,20 +277,26 @@ The high-level API provides a simple managed interface through the [`Llama`](htt
 Below is a short example demonstrating how to use the high-level API to for basic text completion:
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(
+from llama_cpp import Llama
+
+llm = Llama(
       model_path="./models/7B/llama-model.gguf",
       # n_gpu_layers=-1, # Uncomment to use GPU acceleration
       # seed=1337, # Uncomment to set a specific seed
       # n_ctx=2048, # Uncomment to increase the context window
 )
->>> output = llm(
+output = llm(
       "Q: Name the planets in the solar system? A: ", # Prompt
       max_tokens=32, # Generate up to 32 tokens, set to None to generate up to the end of the context window
       stop=["Q:", "\n"], # Stop generating just before the model would generate a new question
       echo=True # Echo the prompt back in the output
 ) # Generate a completion, can also call create_completion
->>> print(output)
+print(output)
+```
+
+By default `llama-cpp-python` generates completions in an OpenAI compatible format:
+
+```python
 {
   "id": "cmpl-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
   "object": "text_completion",
@@ -344,12 +351,12 @@ The model will will format the messages into a single prompt using the following
 Set `verbose=True` to see the selected chat format.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(
+from llama_cpp import Llama
+llm = Llama(
       model_path="path/to/llama-2/llama-model.gguf",
       chat_format="llama-2"
 )
->>> llm.create_chat_completion(
+llm.create_chat_completion(
       messages = [
           {"role": "system", "content": "You are an assistant who perfectly describes images."},
           {
@@ -374,9 +381,9 @@ To constrain chat responses to only valid JSON or a specific JSON Schema use the
 The following example will constrain the response to valid JSON strings only.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
->>> llm.create_chat_completion(
+from llama_cpp import Llama
+llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
+llm.create_chat_completion(
     messages=[
         {
             "role": "system",
@@ -396,9 +403,9 @@ The following example will constrain the response to valid JSON strings only.
 To constrain the response further to a specific JSON Schema add the schema to the `schema` property of the `response_format` argument.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
->>> llm.create_chat_completion(
+from llama_cpp import Llama
+llm = Llama(model_path="path/to/model.gguf", chat_format="chatml")
+llm.create_chat_completion(
     messages=[
         {
             "role": "system",
@@ -423,9 +430,9 @@ To constrain the response further to a specific JSON Schema add the schema to th
 The high-level API supports OpenAI compatible function and tool calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format.
 
 ```python
->>> from llama_cpp import Llama
->>> llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling")
->>> llm.create_chat_completion(
+from llama_cpp import Llama
+llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling")
+llm.create_chat_completion(
       messages = [
         {
           "role": "system",
@@ -475,9 +482,9 @@ The various gguf-converted files for this set of models can be found [here](http
 Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is required to provide HF Tokenizer for functionary. The `LlamaHFTokenizer` class can be initialized and passed into the Llama class. This will override the default llama.cpp tokenizer used in Llama class. The tokenizer files are already included in the respective HF repositories hosting the gguf files.
 
 ```python
->>> from llama_cpp import Llama
->>> from llama_cpp.llama_tokenizer import LlamaHFTokenizer
->>> llm = Llama.from_pretrained(
+from llama_cpp import Llama
+from llama_cpp.llama_tokenizer import LlamaHFTokenizer
+llm = Llama.from_pretrained(
   repo_id="meetkai/functionary-small-v2.2-GGUF",
   filename="functionary-small-v2.2.q4_0.gguf",
   chat_format="functionary-v2",
@@ -503,15 +510,15 @@ You'll first need to download one of the available multi-modal models in GGUF fo
 Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images.
 
 ```python
->>> from llama_cpp import Llama
->>> from llama_cpp.llama_chat_format import Llava15ChatHandler
->>> chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin")
->>> llm = Llama(
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import Llava15ChatHandler
+chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin")
+llm = Llama(
   model_path="./path/to/llava/llama-model.gguf",
   chat_handler=chat_handler,
   n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
 )
->>> llm.create_chat_completion(
+llm.create_chat_completion(
     messages = [
         {"role": "system", "content": "You are an assistant who perfectly describes images."},
         {
@@ -528,19 +535,22 @@ Then you'll need to use a custom chat handler to load the clip model and process
 You can also pull the model from the Hugging Face Hub using the `from_pretrained` method.
 
 ```python
->>> from llama_cpp import Llama
->>> from llama_cpp.llama_chat_format import MoondreamChatHandler
->>> chat_handler = MoondreamChatHandler.from_pretrained(
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import MoondreamChatHandler
+
+chat_handler = MoondreamChatHandler.from_pretrained(
   repo_id="vikhyatk/moondream2",
   filename="*mmproj*",
 )
->>> llm = Llama.from_pretrained(
-  repo_id="vikhyatk/moondream2"
+
+llm = Llama.from_pretrained(
+  repo_id="vikhyatk/moondream2",
   filename="*text-model*",
   chat_handler=chat_handler,
   n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
 )
->>> llm.create_chat_completion(
+
+respoonse = llm.create_chat_completion(
     messages = [
         {
             "role": "user",
@@ -552,6 +562,7 @@ You can also pull the model from the Hugging Face Hub using the `from_pretrained
         }
     ]
 )
+print(response["choices"][0]["text"])
 ```
 
 **Note**: Multi-modal models also support tool calling and JSON mode.
@@ -704,18 +715,18 @@ The entire low-level API can be found in [llama_cpp/llama_cpp.py](https://github
 Below is a short example demonstrating how to use the low-level API to tokenize a prompt:
 
 ```python
->>> import llama_cpp
->>> import ctypes
->>> llama_cpp.llama_backend_init(False) # Must be called once at the start of each program
->>> params = llama_cpp.llama_context_default_params()
+import llama_cpp
+import ctypes
+llama_cpp.llama_backend_init(False) # Must be called once at the start of each program
+params = llama_cpp.llama_context_default_params()
 # use bytes for char * params
->>> model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params)
->>> ctx = llama_cpp.llama_new_context_with_model(model, params)
->>> max_tokens = params.n_ctx
+model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params)
+ctx = llama_cpp.llama_new_context_with_model(model, params)
+max_tokens = params.n_ctx
 # use ctypes arrays for array params
->>> tokens = (llama_cpp.llama_token * int(max_tokens))()
->>> n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True))
->>> llama_cpp.llama_free(ctx)
+tokens = (llama_cpp.llama_token * int(max_tokens))()
+n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True))
+llama_cpp.llama_free(ctx)
 ```
 
 Check out the [examples folder](examples/low_level_api) for more examples of using the low-level API.

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.67"
+__version__ = "0.2.69"

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/llama_cpp/_internals.py

@@ -15,6 +15,7 @@ import numpy.typing as npt
 
 from .llama_types import *
 from .llama_grammar import LlamaGrammar
+from ._utils import suppress_stdout_stderr
 
 import llama_cpp.llama_cpp as llama_cpp
 
@@ -47,9 +48,10 @@ class _LlamaModel:
         if not os.path.exists(path_model):
             raise ValueError(f"Model path does not exist: {path_model}")
 
-        self.model = llama_cpp.llama_load_model_from_file(
-            self.path_model.encode("utf-8"), self.params
-        )
+        with suppress_stdout_stderr(disable=verbose):
+            self.model = llama_cpp.llama_load_model_from_file(
+                self.path_model.encode("utf-8"), self.params
+            )
 
         if self.model is None:
             raise ValueError(f"Failed to load model from file: {path_model}")

{llama_cpp_python-0.2.67 → llama_cpp_python-0.2.69}/llama_cpp/_utils.py

@@ -1,13 +1,15 @@
 import os
 import sys
 
-import sys
 from typing import Any, Dict
 
 # Avoid "LookupError: unknown encoding: ascii" when open() called in a destructor
 outnull_file = open(os.devnull, "w")
 errnull_file = open(os.devnull, "w")
 
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
 class suppress_stdout_stderr(object):
     # NOTE: these must be "saved" here to avoid exceptions when using
     #       this context manager inside of a __del__ method
@@ -22,12 +24,8 @@ class suppress_stdout_stderr(object):
         if self.disable:
             return self
 
-        # Check if sys.stdout and sys.stderr have fileno method
-        if not hasattr(self.sys.stdout, 'fileno') or not hasattr(self.sys.stderr, 'fileno'):
-            return self  # Return the instance without making changes
-
-        self.old_stdout_fileno_undup = self.sys.stdout.fileno()
-        self.old_stderr_fileno_undup = self.sys.stderr.fileno()
+        self.old_stdout_fileno_undup = STDOUT_FILENO
+        self.old_stderr_fileno_undup = STDERR_FILENO
 
         self.old_stdout_fileno = self.os.dup(self.old_stdout_fileno_undup)
         self.old_stderr_fileno = self.os.dup(self.old_stderr_fileno_undup)
@@ -47,15 +45,14 @@ class suppress_stdout_stderr(object):
             return
 
         # Check if sys.stdout and sys.stderr have fileno method
-        if hasattr(self.sys.stdout, 'fileno') and hasattr(self.sys.stderr, 'fileno'):
-            self.sys.stdout = self.old_stdout
-            self.sys.stderr = self.old_stderr
+        self.sys.stdout = self.old_stdout
+        self.sys.stderr = self.old_stderr
 
-            self.os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup)
-            self.os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup)
+        self.os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup)
+        self.os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup)
 
-            self.os.close(self.old_stdout_fileno)
-            self.os.close(self.old_stderr_fileno)
+        self.os.close(self.old_stdout_fileno)
+        self.os.close(self.old_stderr_fileno)
 
 
 class MetaSingleton(type):