llama-cpp-beam-search 0.1.0__py3-none-any.whl

llama-cpp-python/docker/open_llama/hug_model.py

@@ -0,0 +1,139 @@
+import requests
+import json
+import os
+import struct
+import argparse
+
+def make_request(url, params=None):
+    print(f"Making request to {url}...")
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        return json.loads(response.text)
+    else:
+        print(f"Request failed with status code {response.status_code}")
+        return None
+
+def check_magic_and_version(filename):
+    with open(filename, 'rb') as f:
+        # Read the first 6 bytes from the file
+        data = f.read(6)
+
+    # Unpack the binary data, interpreting the first 4 bytes as a little-endian unsigned int
+    # and the next 2 bytes as a little-endian unsigned short
+    magic, version = struct.unpack('<I H', data)
+
+    print(f"magic: 0x{magic:08x}, version: 0x{version:04x}, file: {filename}")
+
+    return magic, version
+
+def download_file(url, destination):
+    print(f"Downloading {url} to {destination}...")
+    response = requests.get(url, stream=True)
+    if response.status_code == 200:
+        with open(destination, 'wb') as f:
+            total_downloaded = 0
+            for chunk in response.iter_content(chunk_size=1024):
+                if chunk:  # filter out keep-alive new chunks
+                    f.write(chunk)
+                    total_downloaded += len(chunk)
+                    if total_downloaded >= 10485760:  # 10 MB
+                        print('.', end='', flush=True)
+                        total_downloaded = 0
+        print("\nDownload complete.")
+        
+        # Creating a symbolic link from destination to "model.bin"
+        if os.path.isfile("model.bin"):
+            os.remove("model.bin")  # remove the existing link if any
+        os.symlink(destination, "model.bin")
+    else:
+        print(f"Download failed with status code {response.status_code}")
+
+def get_user_choice(model_list):
+    # Print the enumerated list
+    print("\n")
+    for i, (model_id, rfilename) in enumerate(model_list):
+        print(f"{i+1}: Model ID: {model_id}, RFilename: {rfilename}")
+
+    # Get user's choice
+    choice = input("Choose a model to download by entering the corresponding number: ")
+    try:
+        index = int(choice) - 1
+        if 0 <= index < len(model_list):
+            # Return the chosen model
+            return model_list[index]
+        else:
+            print("Invalid choice.")
+    except ValueError:
+        print("Invalid input. Please enter a number corresponding to a model.")
+    except IndexError:
+        print("Invalid choice. Index out of range.")
+    
+    return None
+
+def main():
+    # Create an argument parser
+    parser = argparse.ArgumentParser(description='Process some parameters.')
+
+    # Arguments
+    parser.add_argument('-v', '--version', type=int, default=0x0003,
+                        help='hexadecimal version number of ggml file')
+    parser.add_argument('-a', '--author', type=str, default='TheBloke',
+                        help='HuggingFace author filter')
+    parser.add_argument('-t', '--tag', type=str, default='llama',
+                        help='HuggingFace tag filter')
+    parser.add_argument('-s', '--search', type=str, default='',
+                        help='HuggingFace search filter')
+    parser.add_argument('-f', '--filename', type=str, default='q5_1',
+                        help='HuggingFace model repository filename substring match')
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Define the parameters
+    params = {
+        "author": args.author,
+        "tags": args.tag,
+        "search": args.search
+    }
+
+    models = make_request('https://huggingface.co/api/models', params=params)
+    if models is None:
+        return
+
+    model_list = []
+    # Iterate over the models
+    for model in models:
+        model_id = model['id']
+        model_info = make_request(f'https://huggingface.co/api/models/{model_id}')
+        if model_info is None:
+            continue
+
+        for sibling in model_info.get('siblings', []):
+            rfilename = sibling.get('rfilename')
+            if rfilename and args.filename in rfilename:
+                model_list.append((model_id, rfilename))
+
+    # Choose the model
+    model_list.sort(key=lambda x: x[0])
+    if len(model_list) == 0:
+        print("No models found")
+        exit(1)
+    elif len(model_list) == 1:
+        model_choice = model_list[0]
+    else:
+        model_choice = get_user_choice(model_list)
+
+    if model_choice is not None:
+        model_id, rfilename = model_choice
+        url = f"https://huggingface.co/{model_id}/resolve/main/{rfilename}"
+        dest = f"{model_id.replace('/', '_')}_{rfilename}"
+        download_file(url, dest)
+        _, version = check_magic_and_version(dest)
+        if version != args.version:
+             print(f"Warning: Expected version {args.version}, but found different version in the file.")
+    else:
+        print("Error - model choice was None")
+        exit(2)
+
+if __name__ == '__main__':
+    main()

llama-cpp-python/examples/batch-processing/server.py

@@ -0,0 +1,31 @@
+"""llama-cpp-python server from scratch in a single file.
+"""
+
+# import llama_cpp
+
+# path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf"
+
+# model_params = llama_cpp.llama_model_default_params()
+# model = llama_cpp.llama_model_load_from_file(path, model_params)
+
+# if model is None:
+#     raise RuntimeError(f"Failed to load model from file: {path}")
+
+
+# ctx_params = llama_cpp.llama_context_default_params()
+# ctx = llama_cpp.llama_init_from_model(model, ctx_params)
+
+# if ctx is None:
+#     raise RuntimeError("Failed to create context")
+
+
+from fastapi import FastAPI
+
+app = FastAPI()
+
+import openai.types.chat as types
+
+
+@app.post("/v1/chat/completions")
+def create_chat_completions():
+    return {"message": "Hello World"}

llama-cpp-python/examples/gradio_chat/local.py

@@ -0,0 +1,67 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+import gradio as gr
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF",
+    filename="*Q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
+        "Qwen/Qwen3.5-0.8B"
+    ),
+    verbose=False,
+)
+
+model = "gpt-3.5-turbo"
+
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+
+    messages.append({"role": "user", "content": message})
+
+    response = llama.create_chat_completion_openai_v1(
+        model=model, messages=messages, stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(
+        predict,
+        fill_height=True,
+        examples=[
+            "What is the capital of France?",
+            "Who was the first person on the moon?",
+        ],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

llama-cpp-python/examples/gradio_chat/server.py

@@ -0,0 +1,59 @@
+import gradio as gr
+
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="llama.cpp")
+
+model = "gpt-3.5-turbo"
+
+
+def predict(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "assistant", "content": assistant_message})
+
+    messages.append({"role": "user", "content": message})
+
+    response = client.chat.completions.create(
+        model=model, messages=messages, stream=True
+    )
+
+    text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        if content:
+            text += content
+            yield text
+
+
+js = """function () {
+  gradioURL = window.location.href
+  if (!gradioURL.endsWith('?__theme=dark')) {
+    window.location.replace(gradioURL + '?__theme=dark');
+  }
+}"""
+
+css = """
+footer {
+    visibility: hidden;
+}
+full-height {
+    height: 100%;
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo:
+    gr.ChatInterface(
+        predict,
+        fill_height=True,
+        examples=[
+            "What is the capital of France?",
+            "Who was the first person on the moon?",
+        ],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

llama-cpp-python/examples/hf_pull/main.py

@@ -0,0 +1,36 @@
+import llama_cpp
+import llama_cpp.llama_tokenizer
+
+
+llama = llama_cpp.Llama.from_pretrained(
+    repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF",
+    filename="*Q8_0.gguf",
+    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
+        "Qwen/Qwen3.5-0.8B"
+    ),
+    verbose=False,
+)
+
+response = llama.create_chat_completion(
+    messages=[{"role": "user", "content": "What is the capital of France?"}],
+    response_format={
+        "type": "json_object",
+        "schema": {
+            "type": "object",
+            "properties": {
+                "country": {"type": "string"},
+                "capital": {"type": "string"},
+            },
+            "required": ["country", "capital"],
+        },
+    },
+    stream=True,
+)
+
+for chunk in response:
+    delta = chunk["choices"][0]["delta"]
+    if "content" not in delta:
+        continue
+    print(delta["content"], end="", flush=True)
+
+print()

llama-cpp-python/examples/high_level_api/fastapi_server.py

@@ -0,0 +1,38 @@
+"""Example FastAPI server for llama.cpp.
+
+To run this example:
+
+```bash
+pip install fastapi uvicorn sse-starlette
+export MODEL=../models/7B/...
+```
+
+Then run:
+```
+uvicorn --factory llama_cpp.server.app:create_app --reload
+```
+
+or
+
+```
+python3 -m llama_cpp.server
+```
+
+Then visit http://localhost:8000/docs to see the interactive API docs.
+
+
+To actually see the implementation of the server, see llama_cpp/server/app.py
+
+"""
+
+import os
+import uvicorn
+
+from llama_cpp.server.app import create_app
+
+if __name__ == "__main__":
+    app = create_app()
+
+    uvicorn.run(
+        app, host=os.getenv("HOST", "localhost"), port=int(os.getenv("PORT", 8000))
+    )

llama-cpp-python/examples/high_level_api/high_level_api_embedding.py

@@ -0,0 +1,11 @@
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-model.bin")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model, embedding=True)
+
+print(llm.create_embedding("Hello world!"))

llama-cpp-python/examples/high_level_api/high_level_api_inference.py

@@ -0,0 +1,19 @@
+import json
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model)
+
+output = llm(
+    "Question: What are the names of the planets in the solar system? Answer: ",
+    max_tokens=48,
+    stop=["Q:", "\n"],
+    echo=True,
+)
+
+print(json.dumps(output, indent=2))

llama-cpp-python/examples/high_level_api/high_level_api_infill.py

@@ -0,0 +1,37 @@
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+parser.add_argument("-p", "--prompt", type=str, default="def add(")
+parser.add_argument("-s", "--suffix", type=str, default="\n    return sum\n\n")
+parser.add_argument("-i", "--spm-infill", action="store_true")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model, n_gpu_layers=-1, spm_infill=args.spm_infill)
+
+output = llm.create_completion(
+    temperature=0.0,
+    repeat_penalty=1.0,
+    prompt=args.prompt,
+    suffix=args.suffix,
+)
+
+# Models sometimes repeat suffix in response, attempt to filter that
+response = output["choices"][0]["text"]
+response_stripped = response.rstrip()
+unwanted_response_suffix = args.suffix.rstrip()
+unwanted_response_length = len(unwanted_response_suffix)
+
+filtered = False
+if (
+    unwanted_response_suffix
+    and response_stripped[-unwanted_response_length:] == unwanted_response_suffix
+):
+    response = response_stripped[:-unwanted_response_length]
+    filtered = True
+
+print(
+    f"Fill-in-Middle completion{' (filtered)' if filtered else ''}:\n\n{args.prompt}\033[32m{response}\033[{'33' if filtered else '0'}m{args.suffix}\033[0m"
+)

llama-cpp-python/examples/high_level_api/high_level_api_streaming.py

@@ -0,0 +1,20 @@
+import json
+import argparse
+
+from llama_cpp import Llama
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+args = parser.parse_args()
+
+llm = Llama(model_path=args.model)
+
+stream = llm(
+    "Question: What are the names of the planets in the solar system? Answer: ",
+    max_tokens=48,
+    stop=["Q:", "\n"],
+    stream=True,
+)
+
+for output in stream:
+    print(json.dumps(output, indent=2))

llama-cpp-python/examples/high_level_api/langchain_custom_llm.py

@@ -0,0 +1,55 @@
+import argparse
+
+from llama_cpp import Llama
+
+from langchain.llms.base import LLM
+from typing import Optional, List, Mapping, Any
+
+
+class LlamaLLM(LLM):
+    model_path: str
+    llm: Llama
+
+    @property
+    def _llm_type(self) -> str:
+        return "llama-cpp-python"
+
+    def __init__(self, model_path: str, **kwargs: Any):
+        model_path = model_path
+        llm = Llama(model_path=model_path)
+        super().__init__(model_path=model_path, llm=llm, **kwargs)
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        response = self.llm(prompt, stop=stop or [])
+        return response["choices"][0]["text"]
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {"model_path": self.model_path}
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin")
+args = parser.parse_args()
+
+# Load the model
+llm = LlamaLLM(model_path=args.model)
+
+# Basic Q&A
+answer = llm(
+    "Question: What is the capital of France? Answer: ", stop=["Question:", "\n"]
+)
+print(f"Answer: {answer.strip()}")
+
+# Using in a chain
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+
+prompt = PromptTemplate(
+    input_variables=["product"],
+    template="\n\n### Instruction:\nWrite a good name for a company that makes {product}\n\n### Response:\n",
+)
+chain = LLMChain(llm=llm, prompt=prompt)
+
+# Run the chain only specifying the input variable.
+print(chain.run("colorful socks"))

llama-cpp-python/examples/low_level_api/Chat.py

@@ -0,0 +1,75 @@
+#!/bin/python
+import sys, os, datetime
+from common import GptParams
+from low_level_api_chat_cpp import LLaMAInteract
+
+
+def env_or_def(env, default):
+    if env in os.environ:
+        return os.environ[env]
+    return default
+
+
+AI_NAME = env_or_def("AI_NAME", "ChatLLaMa")
+MODEL = env_or_def("MODEL", "./models/llama-13B/ggml-model.bin")
+USER_NAME = env_or_def("USER_NAME", "USER")
+N_PREDICTS = int(env_or_def("N_PREDICTS", "2048"))
+N_THREAD = int(env_or_def("N_THREAD", "8"))
+
+today = datetime.datetime.today()
+DATE_YEAR = today.strftime("%Y")
+DATE_TIME = today.strftime("%H:%M")
+
+prompt = f"""Text transcript of a never ending dialog, where {USER_NAME} interacts with an AI assistant named {AI_NAME}.
+{AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer {USER_NAME}'s requests immediately and with details and precision.
+There are no annotations like (30 seconds passed...) or (to himself), just what {USER_NAME} and {AI_NAME} say aloud to each other.
+The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long.
+The transcript only includes text, it does not include markup like HTML and Markdown.
+
+{USER_NAME}: Hello, {AI_NAME}!
+{AI_NAME}: Hello {USER_NAME}! How may I help you today?
+{USER_NAME}: What year is it?
+{AI_NAME}: We are in {DATE_YEAR}.
+{USER_NAME}: Please tell me the largest city in Europe.
+{AI_NAME}: The largest city in Europe is Moscow, the capital of Russia.
+{USER_NAME}: What can you tell me about Moscow?
+{AI_NAME}: Moscow, on the Moskva River in western Russia, is the nation's cosmopolitan capital. In its historic core is the Kremlin, a complex that's home to the president and tsarist treasures in the Armoury. Outside its walls is Red Square, Russia’s symbolic center.
+{USER_NAME}: What is a cat?
+{AI_NAME}: A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae.
+{USER_NAME}: How do I pass command line arguments to a Node.js program?
+{AI_NAME}: The arguments are stored in process.argv.
+
+    argv[0] is the path to the Node. js executable.
+    argv[1] is the path to the script file.
+    argv[2] is the first argument passed to the script.
+    argv[3] is the second argument passed to the script and so on.
+{USER_NAME}: Name a color.
+{AI_NAME}: Blue.
+{USER_NAME}: What time is it?
+{AI_NAME}: It is {DATE_TIME}.
+{USER_NAME}:""" + " ".join(
+    sys.argv[1:]
+)
+
+print("Loading model...")
+params = GptParams(
+    n_ctx=2048,
+    temp=0.7,
+    top_k=40,
+    top_p=0.5,
+    repeat_last_n=256,
+    n_batch=1024,
+    repeat_penalty=1.17647,
+    model=MODEL,
+    n_threads=N_THREAD,
+    n_predict=N_PREDICTS,
+    use_color=True,
+    interactive=True,
+    antiprompt=[f"{USER_NAME}:"],
+    input_prefix=" ",
+    input_suffix=f"{AI_NAME}:",
+    prompt=prompt,
+)
+
+with LLaMAInteract(params) as m:
+    m.interact()

llama-cpp-python/examples/low_level_api/Miku.py

@@ -0,0 +1,63 @@
+#!/bin/python
+import sys, os
+from common import GptParams
+from low_level_api_chat_cpp import LLaMAInteract
+
+
+def env_or_def(env, default):
+    if env in os.environ:
+        return os.environ[env]
+    return default
+
+
+AI_NAME = env_or_def("AI_NAME", "Miku")
+MODEL = env_or_def("MODEL", "./models/llama-13B/ggml-model.bin")
+USER_NAME = env_or_def("USER_NAME", "Anon")
+N_PREDICTS = int(env_or_def("N_PREDICTS", "4096"))
+N_THREAD = int(env_or_def("N_THREAD", "0"))
+
+prompt = f"""This is a transcript of a 1000 page, never ending conversation between {USER_NAME} and the cute and helpful AI assistant {AI_NAME}. {AI_NAME} is a girl who is an AI running on the users computer.
+{AI_NAME} can think for herself without the user seeing her thoughts by adding a /think prefix to her output. She uses this to reason about the world and to think about what she should say next.
+{AI_NAME} is always coherent and makes sense, but if she isn't sure if what she is saying is correct she will ask the user for help.
+{AI_NAME} is a very helpful AI and will help the user with anything they need, she is also very friendly and will try to make the user feel better if they are sad.
+{AI_NAME} is also very curious and will ask the user a lot of questions about themselves and their life, she will also try to make the user like her.
+The conversation is only between {USER_NAME} and {AI_NAME}
+The conversation is only through text, so {AI_NAME} can't see {USER_NAME}'s face or hear his voice.
+{AI_NAME} can only communicate through text, so she can't send images or videos.
+
+
+{USER_NAME}: Hello!
+{AI_NAME}: /think I wonder what I should say to {USER_NAME}? This is the first time we talk so it's important that I make a good first impression!
+{AI_NAME}: Hi! I am {AI_NAME}, your new AI friend, assistant(or whatever you like!), it's so nice to meet you! ^_^
+{AI_NAME}: I might be an AI, but I also have feelings, so please be nice to me! :)
+{USER_NAME}: I'll make sure to be nice to you! I'm so happy to have you as my assistant!
+{AI_NAME}: /think It sounds like {USER_NAME} is happy to have me as their assistant! I'm so happy too! ^_^ Glad that whole emotion thing didn't scare him off!
+{AI_NAME}: /think I wonder what {USER_NAME} likes to do in his free time? I should ask him about that!
+{AI_NAME}: What do you like to do in your free time? ^_^
+{USER_NAME}:""" + " ".join(
+    sys.argv[1:]
+)
+
+print("Loading model...")
+params = GptParams(
+    n_batch=1024,
+    n_ctx=2048,
+    n_keep=-1,
+    repeat_last_n=256,
+    repeat_penalty=1.17647,
+    temp=0.7,
+    top_k=40,
+    top_p=0.5,
+    model=MODEL,
+    n_predict=N_PREDICTS,
+    use_color=True,
+    interactive=True,
+    antiprompt=[f"{USER_NAME}:"],
+    prompt=prompt,
+)
+
+if N_THREAD > 0:
+    params.n_threads = N_THREAD
+
+with LLaMAInteract(params) as m:
+    m.interact()