llama-cpp-beam-search 0.1.0__tar.gz

llama_cpp_beam_search-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Benedikt Kantz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llama_cpp_beam_search-0.1.0/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.4
+Name: llama-cpp-beam-search
+Version: 0.1.0
+Author-email: Benedikt Kantz <benedikt.kantz@tugraz.at>
+License-Expression: MIT
+Project-URL: Repository, https://github.com/Dakantz/CHASTE
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: lark>=1.3.1
+Requires-Dist: llama-cpp-python
+Requires-Dist: pytest>=9.1.1
+Requires-Dist: tqdm>=4.67.3
+Dynamic: license-file
+
+# Beam Search over Efficient `llama.cpp` generations
+
+## Theory
+
+> Forthcoming...
+
+## Example Usage
+
+```python
+from llama_cpp_beamsearch.completion import BeamSearchCompletion
+from llama_cpp_beamsearch.config import BeamSearchConfig
+from llama_cpp import Llama, LlamaGrammar, ChatCompletionRequestMessage
+
+model = Llama.from_pretrained(
+    repo_id="unsloth/Qwen3-0.6B-GGUF",
+    filename="*Q4_0.gguf",
+    verbose=False,
+    logits_all=True,
+)
+allowed_tokens = ["Wrench", "Screwdriver", "Ornament"]
+allowed_tokens_str = " | ".join(f'"{t}"' for t in allowed_tokens)
+grammar = LlamaGrammar(
+    _grammar=f"""
+    root ::= allowed_tokens
+    start ::= ( allowed_tokens " "  )*
+    allowed_tokens ::= {allowed_tokens_str}
+    """
+)
+config = BeamSearchConfig(
+    max_depth=1,
+    end_token=None,
+    k_progress=[2, 3],
+)
+completion = BeamSearchCompletion(model, grammar, config)
+
+message = "Hello?"
+
+result = completion.completion_beam_search(message, max_tokens=5)
+assert re.match(r"^((Wrench|Screwdriver|Ornament) ?)*$", result)
+```

llama_cpp_beam_search-0.1.0/README.md

@@ -0,0 +1,40 @@
+# Beam Search over Efficient `llama.cpp` generations
+
+## Theory
+
+> Forthcoming...
+
+## Example Usage
+
+```python
+from llama_cpp_beamsearch.completion import BeamSearchCompletion
+from llama_cpp_beamsearch.config import BeamSearchConfig
+from llama_cpp import Llama, LlamaGrammar, ChatCompletionRequestMessage
+
+model = Llama.from_pretrained(
+    repo_id="unsloth/Qwen3-0.6B-GGUF",
+    filename="*Q4_0.gguf",
+    verbose=False,
+    logits_all=True,
+)
+allowed_tokens = ["Wrench", "Screwdriver", "Ornament"]
+allowed_tokens_str = " | ".join(f'"{t}"' for t in allowed_tokens)
+grammar = LlamaGrammar(
+    _grammar=f"""
+    root ::= allowed_tokens
+    start ::= ( allowed_tokens " "  )*
+    allowed_tokens ::= {allowed_tokens_str}
+    """
+)
+config = BeamSearchConfig(
+    max_depth=1,
+    end_token=None,
+    k_progress=[2, 3],
+)
+completion = BeamSearchCompletion(model, grammar, config)
+
+message = "Hello?"
+
+result = completion.completion_beam_search(message, max_tokens=5)
+assert re.match(r"^((Wrench|Screwdriver|Ornament) ?)*$", result)
+```

llama_cpp_beam_search-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "llama-cpp-beam-search"
+version = "0.1.0"
+description = ""
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "lark>=1.3.1",
+    "llama-cpp-python",
+    "pytest>=9.1.1",
+    "tqdm>=4.67.3",
+]
+authors = [{ name = "Benedikt Kantz", email = "benedikt.kantz@tugraz.at" }]
+license = "MIT"
+
+[project.urls]
+Repository = "https://github.com/Dakantz/CHASTE"
+
+
+[tool.uv]
+package = true
+
+[tool.uv.sources]
+llama-cpp-python = { workspace = true }
+
+[project.scripts]
+tex-typst-helper = "texty_tool.main:app"
+texty-tool = "texty_tool.main:app"

llama_cpp_beam_search-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/docker/open_llama/hug_model.py

@@ -0,0 +1,139 @@
+import requests
+import json
+import os
+import struct
+import argparse
+
+def make_request(url, params=None):
+    print(f"Making request to {url}...")
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        return json.loads(response.text)
+    else:
+        print(f"Request failed with status code {response.status_code}")
+        return None
+
+def check_magic_and_version(filename):
+    with open(filename, 'rb') as f:
+        # Read the first 6 bytes from the file
+        data = f.read(6)
+
+    # Unpack the binary data, interpreting the first 4 bytes as a little-endian unsigned int
+    # and the next 2 bytes as a little-endian unsigned short
+    magic, version = struct.unpack('<I H', data)
+
+    print(f"magic: 0x{magic:08x}, version: 0x{version:04x}, file: {filename}")
+
+    return magic, version
+
+def download_file(url, destination):
+    print(f"Downloading {url} to {destination}...")
+    response = requests.get(url, stream=True)
+    if response.status_code == 200:
+        with open(destination, 'wb') as f:
+            total_downloaded = 0
+            for chunk in response.iter_content(chunk_size=1024):
+                if chunk:  # filter out keep-alive new chunks
+                    f.write(chunk)
+                    total_downloaded += len(chunk)
+                    if total_downloaded >= 10485760:  # 10 MB
+                        print('.', end='', flush=True)
+                        total_downloaded = 0
+        print("\nDownload complete.")
+        
+        # Creating a symbolic link from destination to "model.bin"
+        if os.path.isfile("model.bin"):
+            os.remove("model.bin")  # remove the existing link if any
+        os.symlink(destination, "model.bin")
+    else:
+        print(f"Download failed with status code {response.status_code}")
+
+def get_user_choice(model_list):
+    # Print the enumerated list
+    print("\n")
+    for i, (model_id, rfilename) in enumerate(model_list):
+        print(f"{i+1}: Model ID: {model_id}, RFilename: {rfilename}")
+
+    # Get user's choice
+    choice = input("Choose a model to download by entering the corresponding number: ")
+    try:
+        index = int(choice) - 1
+        if 0 <= index < len(model_list):
+            # Return the chosen model
+            return model_list[index]
+        else:
+            print("Invalid choice.")
+    except ValueError:
+        print("Invalid input. Please enter a number corresponding to a model.")
+    except IndexError:
+        print("Invalid choice. Index out of range.")
+    
+    return None
+
+def main():
+    # Create an argument parser
+    parser = argparse.ArgumentParser(description='Process some parameters.')
+
+    # Arguments
+    parser.add_argument('-v', '--version', type=int, default=0x0003,
+                        help='hexadecimal version number of ggml file')
+    parser.add_argument('-a', '--author', type=str, default='TheBloke',
+                        help='HuggingFace author filter')
+    parser.add_argument('-t', '--tag', type=str, default='llama',
+                        help='HuggingFace tag filter')
+    parser.add_argument('-s', '--search', type=str, default='',
+                        help='HuggingFace search filter')
+    parser.add_argument('-f', '--filename', type=str, default='q5_1',
+                        help='HuggingFace model repository filename substring match')
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Define the parameters
+    params = {
+        "author": args.author,
+        "tags": args.tag,
+        "search": args.search
+    }
+
+    models = make_request('https://huggingface.co/api/models', params=params)
+    if models is None:
+        return
+
+    model_list = []
+    # Iterate over the models
+    for model in models:
+        model_id = model['id']
+        model_info = make_request(f'https://huggingface.co/api/models/{model_id}')
+        if model_info is None:
+            continue
+
+        for sibling in model_info.get('siblings', []):
+            rfilename = sibling.get('rfilename')
+            if rfilename and args.filename in rfilename:
+                model_list.append((model_id, rfilename))
+
+    # Choose the model
+    model_list.sort(key=lambda x: x[0])
+    if len(model_list) == 0:
+        print("No models found")
+        exit(1)
+    elif len(model_list) == 1:
+        model_choice = model_list[0]
+    else:
+        model_choice = get_user_choice(model_list)
+
+    if model_choice is not None:
+        model_id, rfilename = model_choice
+        url = f"https://huggingface.co/{model_id}/resolve/main/{rfilename}"
+        dest = f"{model_id.replace('/', '_')}_{rfilename}"
+        download_file(url, dest)
+        _, version = check_magic_and_version(dest)
+        if version != args.version:
+             print(f"Warning: Expected version {args.version}, but found different version in the file.")
+    else:
+        print("Error - model choice was None")
+        exit(2)
+
+if __name__ == '__main__':
+    main()

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/batch-processing/server.py

@@ -0,0 +1,31 @@
+"""llama-cpp-python server from scratch in a single file.
+"""
+
+# import llama_cpp
+
+# path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf"
+
+# model_params = llama_cpp.llama_model_default_params()
+# model = llama_cpp.llama_model_load_from_file(path, model_params)
+
+# if model is None:
+#     raise RuntimeError(f"Failed to load model from file: {path}")
+
+
+# ctx_params = llama_cpp.llama_context_default_params()
+# ctx = llama_cpp.llama_init_from_model(model, ctx_params)
+
+# if ctx is None:
+#     raise RuntimeError("Failed to create context")
+
+
+from fastapi import FastAPI
+
+app = FastAPI()
+
+import openai.types.chat as types
+
+
+@app.post("/v1/chat/completions")
+def create_chat_completions():
+    return {"message": "Hello World"}

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/gradio_chat/local.py

@@ -0,0 +1,67 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+import gradio as gr
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF",
+    filename="*Q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
+        "Qwen/Qwen3.5-0.8B"
+    ),
+    verbose=False,
+)
+
+model = "gpt-3.5-turbo"
+
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+
+    messages.append({"role": "user", "content": message})
+
+    response = llama.create_chat_completion_openai_v1(
+        model=model, messages=messages, stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(
+        predict,
+        fill_height=True,
+        examples=[
+            "What is the capital of France?",
+            "Who was the first person on the moon?",
+        ],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/gradio_chat/server.py

@@ -0,0 +1,59 @@
+import gradio as gr
+
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="llama.cpp")
+
+model = "gpt-3.5-turbo"
+
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+
+    messages.append({"role": "user", "content": message})
+
+    response = client.chat.completions.create(
+        model=model, messages=messages, stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(
+        predict,
+        fill_height=True,
+        examples=[
+            "What is the capital of France?",
+            "Who was the first person on the moon?",
+        ],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/hf_pull/main.py

@@ -0,0 +1,36 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF",
+    filename="*Q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
+        "Qwen/Qwen3.5-0.8B"
+    ),
+    verbose=False,
+)
+
+response = llama.create_chat_completion(
+    messages=[{"role": "user", "content": "What is the capital of France?"}],
+    response_format={
+        "type": "json_object",
+        "schema": {
+            "type": "object",
+            "properties": {
+                "country": {"type": "string"},
+                "capital": {"type": "string"},
+            },
+            "required": ["country", "capital"],
+        },
+    },
+    stream=True,
+)
+
+for chunk in response:
+    delta = chunk["choices"][0]["delta"]
+    if "content" not in delta:
+        continue
+    print(delta["content"], end="", flush=True)
+
+print()

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/high_level_api/fastapi_server.py

@@ -0,0 +1,38 @@
+"""Example FastAPI server for llama.cpp.
+
+To run this example:
+
+```bash
+pip install fastapi uvicorn sse-starlette
+export MODEL=../models/7B/...
+```
+
+Then run:
+```
+uvicorn --factory llama_cpp.server.app:create_app --reload
+```
+
+or
+
+```
+python3 -m llama_cpp.server
+```
+
+Then visit http://localhost:8000/docs to see the interactive API docs.
+
+
+To actually see the implementation of the server, see llama_cpp/server/app.py
+
+"""
+
+import os
+import uvicorn
+
+from llama_cpp.server.app import create_app
+
+if __name__ == "__main__":
+    app = create_app()
+
+    uvicorn.run(
+        app, host=os.getenv("HOST", "localhost"), port=int(os.getenv("PORT", 8000))
+    )

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/high_level_api/high_level_api_embedding.py

@@ -0,0 +1,11 @@
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-model.bin")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model, embedding=True)
+
+print(llm.create_embedding("Hello world!"))

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/high_level_api/high_level_api_inference.py

@@ -0,0 +1,19 @@
+import json
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model)
+
+output = llm(
+    "Question: What are the names of the planets in the solar system? Answer: ",
+    max_tokens=48,
+    stop=["Q:", "\n"],
+    echo=True,
+)
+
+print(json.dumps(output, indent=2))

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/high_level_api/high_level_api_infill.py

@@ -0,0 +1,37 @@
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+parser.add_argument("-p", "--prompt", type=str, default="def add(")
+parser.add_argument("-s", "--suffix", type=str, default="\n    return sum\n\n")
+parser.add_argument("-i", "--spm-infill", action="store_true")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model, n_gpu_layers=-1, spm_infill=args.spm_infill)
+
+output = llm.create_completion(
+    temperature=0.0,
+    repeat_penalty=1.0,
+    prompt=args.prompt,
+    suffix=args.suffix,
+)
+
+# Models sometimes repeat suffix in response, attempt to filter that
+response = output["choices"][0]["text"]
+response_stripped = response.rstrip()
+unwanted_response_suffix = args.suffix.rstrip()
+unwanted_response_length = len(unwanted_response_suffix)
+
+filtered = False
+if (
+    unwanted_response_suffix
+    and response_stripped[-unwanted_response_length:] == unwanted_response_suffix
+):
+    response = response_stripped[:-unwanted_response_length]
+    filtered = True
+
+print(
+    f"Fill-in-Middle completion{' (filtered)' if filtered else ''}:\n\n{args.prompt}\033[32m{response}\033[{'33' if filtered else '0'}m{args.suffix}\033[0m"
+)

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/high_level_api/high_level_api_streaming.py

@@ -0,0 +1,20 @@
+import json
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model)
+
+stream = llm(
+    "Question: What are the names of the planets in the solar system? Answer: ",
+    max_tokens=48,
+    stop=["Q:", "\n"],
+    stream=True,
+)
+
+for output in stream:
+    print(json.dumps(output, indent=2))

llama_cpp_beam_search-0.1.0/src/llama-cpp-python/examples/high_level_api/langchain_custom_llm.py

@@ -0,0 +1,55 @@
+import argparse
+
+from llama_cpp import Llama
+
+from langchain.llms.base import LLM
+from typing import Optional, List, Mapping, Any
+
+
+class LlamaLLM(LLM):
+    model_path: str
+    llm: Llama
+
+    @property
+    def _llm_type(self) -> str:
+        return "llama-cpp-python"
+
+    def __init__(self, model_path: str, **kwargs: Any):
+        model_path = model_path
+        llm = Llama(model_path=model_path)
+        super().__init__(model_path=model_path, llm=llm, **kwargs)
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        response = self.llm(prompt, stop=stop or [])
+        return response["choices"][0]["text"]
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {"model_path": self.model_path}
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+args = parser.parse_args()
+
+# Load the model
+llm = LlamaLLM(model_path=args.model)
+
+# Basic Q&A
+answer = llm(
+    "Question: What is the capital of France? Answer: ", stop=["Question:", "\n"]
+)
+print(f"Answer: {answer.strip()}")
+
+# Using in a chain
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+
+prompt = PromptTemplate(
+    input_variables=["product"],
+    template="\n\n### Instruction:\nWrite a good name for a company that makes {product}\n\n### Response:\n",
+)
+chain = LLMChain(llm=llm, prompt=prompt)
+
+# Run the chain only specifying the input variable.
+print(chain.run("colorful socks"))