llama-cpp-python 0.2.50__tar.gz → 0.2.51__tar.gz

llama_cpp_python-0.2.51/.git/FETCH_HEAD

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5		'5fc4c1efb61883087e25c06100e258ca2e7d15f5' of https://github.com/abetlen/llama-cpp-python

llama_cpp_python-0.2.51/.git/HEAD

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/.git/config

@@ -9,7 +9,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3piQXpTYWwzUGd0a05ORThtSE1wTzRWaHJBNjVrNTNkV3pZaA==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hnTUJRc3BOZnpreWs3cnlrdHBtbjQ1MDUxS0dEQzFJeUU0ZQ==
 [submodule "vendor/llama.cpp"]
 	active = true
 	url = https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.51/.git/logs/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 5fc4c1efb61883087e25c06100e258ca2e7d15f5 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913844 +0000	checkout: moving from master to refs/tags/v0.2.51

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/HEAD

@@ -0,0 +1 @@
+c39373398803c669056304090050fe3f44b41bf9

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/.git/modules/vendor/llama.cpp/config

@@ -16,7 +16,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3piQXpTYWwzUGd0a05ORThtSE1wTzRWaHJBNjVrNTNkV3pZaA==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hnTUJRc3BOZnpreWs3cnlrdHBtbjQ1MDUxS0dEQzFJeUU0ZQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/ggerganov/llama.cpp.git
+c39373398803c669056304090050fe3f44b41bf9 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	checkout: moving from master to c39373398803c669056304090050fe3f44b41bf9

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 c39373398803c669056304090050fe3f44b41bf9 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/ggerganov/llama.cpp.git

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/.git/modules/vendor/llama.cpp/modules/kompute/config

@@ -13,7 +13,7 @@
 [gc]
 	auto = 0
 [http "https://github.com/"]
-	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3piQXpTYWwzUGd0a05ORThtSE1wTzRWaHJBNjVrNTNkV3pZaA==
+	extraheader = AUTHORIZATION: basic eC1hY2Nlc3MtdG9rZW46Z2hzX3hnTUJRc3BOZnpreWs3cnlrdHBtbjQ1MDUxS0dEQzFJeUU0ZQ==
 [url "https://github.com/"]
 	insteadOf = git@github.com:
 	insteadOf = org-6826477@github.com:

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/modules/kompute/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/nomic-ai/kompute.git
+d1e3b0953cf66acc94b2e29693e221427b2c1f3f 4565194ed7c32d1d2efa32ceab4d3c6cae006306 runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913846 +0000	checkout: moving from master to 4565194ed7c32d1d2efa32ceab4d3c6cae006306

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/heads/master

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/modules/kompute/logs/refs/remotes/origin/HEAD

@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 d1e3b0953cf66acc94b2e29693e221427b2c1f3f runner <runner@fv-az1152-212.ykhc0jfl13aufdxrnlto1juhnc.cx.internal.cloudapp.net> 1708913845 +0000	clone: from https://github.com/nomic-ai/kompute.git

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled sorted 
+c39373398803c669056304090050fe3f44b41bf9 refs/remotes/origin/master

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/refs/heads/master

@@ -0,0 +1 @@
+c39373398803c669056304090050fe3f44b41bf9

llama_cpp_python-0.2.51/.git/modules/vendor/llama.cpp/shallow

@@ -0,0 +1 @@
+c39373398803c669056304090050fe3f44b41bf9

llama_cpp_python-0.2.51/.git/refs/tags/v0.2.51

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5

llama_cpp_python-0.2.51/.git/shallow

@@ -0,0 +1 @@
+5fc4c1efb61883087e25c06100e258ca2e7d15f5

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.51]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@c39373398803c669056304090050fe3f44b41bf9
+- fix: Restore type hints for low-level api by @abetlen in 19234aa0dbd0c3c87656e65dd2b064665371925b
+
 ## [0.2.50]
 
 - docs: Update Functionary OpenAI Server Readme by @jeffrey-fong in #1193

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama_cpp_python
-Version: 0.2.50
+Version: 0.2.51
 Summary: Python bindings for the llama.cpp library
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT

llama_cpp_python-0.2.51/examples/gradio_chat/local.py

@@ -0,0 +1,59 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+import gradio as gr
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
+    filename="*q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"),
+    verbose=False
+)
+
+model = "gpt-3.5-turbo"
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+    
+    messages.append({"role": "user", "content": message})
+
+    response = llama.create_chat_completion_openai_v1(
+        model=model,
+        messages=messages,
+        stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(predict, fill_height=True, examples=["What is the capital of France?", "Who was the first person on the moon?"])
+
+
+if __name__ == "__main__":
+    demo.launch()

llama_cpp_python-0.2.51/examples/gradio_chat/server.py

@@ -0,0 +1,56 @@
+import gradio as gr
+
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://localhost:8000/v1",
+    api_key="llama.cpp"
+)
+
+model = "gpt-3.5-turbo"
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+    
+    messages.append({"role": "user", "content": message})
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(predict, fill_height=True, examples=["What is the capital of France?", "Who was the first person on the moon?"])
+
+
+if __name__ == "__main__":
+    demo.launch()

llama_cpp_python-0.2.51/examples/hf_pull/main.py

@@ -0,0 +1,39 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
+    filename="*q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"),
+    verbose=False
+)
+
+response = llama.create_chat_completion(
+    messages=[
+        {
+            "role": "user",
+            "content": "What is the capital of France?"
+        }
+    ],
+    response_format={
+        "type": "json_object",
+        "schema": {
+            "type": "object",
+            "properties": {
+                "country": {"type": "string"},
+                "capital": {"type": "string"}
+            },
+            "required": ["country", "capital"],
+        }
+    },
+    stream=True
+)
+
+for chunk in response:
+    delta = chunk["choices"][0]["delta"]
+    if "content" not in delta:
+        continue
+    print(delta["content"], end="", flush=True)
+
+print()

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/llama_cpp/__init__.py

@@ -1,4 +1,4 @@
 from .llama_cpp import *
 from .llama import *
 
-__version__ = "0.2.50"
+__version__ = "0.2.51"

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/llama_cpp/_internals.py

@@ -291,7 +291,7 @@ class _LlamaContext:
 
     def kv_cache_seq_shift(self, seq_id: int, p0: int, p1: int, shift: int):
         assert self.ctx is not None
-        llama_cpp.llama_kv_cache_seq_shift(self.ctx, seq_id, p0, p1, shift)
+        llama_cpp.llama_kv_cache_seq_add(self.ctx, seq_id, p0, p1, shift)
 
     def get_state_size(self) -> int:
         assert self.ctx is not None

{llama_cpp_python-0.2.50 → llama_cpp_python-0.2.51}/llama_cpp/llama.py

@@ -65,7 +65,7 @@ class Llama:
         *,
         # Model Params
         n_gpu_layers: int = 0,
-        split_mode: int = llama_cpp.LLAMA_SPLIT_LAYER,
+        split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER,
         main_gpu: int = 0,
         tensor_split: Optional[List[float]] = None,
         vocab_only: bool = False,
@@ -78,7 +78,7 @@ class Llama:
         n_batch: int = 512,
         n_threads: Optional[int] = None,
         n_threads_batch: Optional[int] = None,
-        rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_UNSPECIFIED,
+        rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
         rope_freq_base: float = 0.0,
         rope_freq_scale: float = 0.0,
         yarn_ext_factor: float = -1.0,
@@ -238,13 +238,13 @@ class Llama:
             for i, (k, v) in enumerate(kv_overrides.items()):
                 self._kv_overrides_array[i].key = k.encode("utf-8")
                 if isinstance(v, bool):
-                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_BOOL
+                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL
                     self._kv_overrides_array[i].value.bool_value = v
                 elif isinstance(v, int):
-                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_INT
+                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT
                     self._kv_overrides_array[i].value.int_value = v
                 elif isinstance(v, float):
-                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_FLOAT
+                    self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT
                     self._kv_overrides_array[i].value.float_value = v
                 else:
                     raise ValueError(f"Unknown value type for {k}: {v}")
@@ -270,7 +270,7 @@ class Llama:
         self.context_params.rope_scaling_type = (
             rope_scaling_type
             if rope_scaling_type is not None
-            else llama_cpp.LLAMA_ROPE_SCALING_UNSPECIFIED
+            else llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED
         )
         self.context_params.rope_freq_base = (
             rope_freq_base if rope_freq_base != 0.0 else 0