llama-cpp-py 0.1.11__tar.gz

llama_cpp_py-0.1.11/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 sergey21000
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llama_cpp_py-0.1.11/PKG-INFO

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: llama-cpp-py
+Version: 0.1.11
+Summary: Python wrapper for running the llama.cpp server
+Author: sergey21000
+License-Expression: MIT
+Project-URL: Repository, https://github.com/sergey21000/llama_cpp_py
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiohttp==3.13.1
+Requires-Dist: requests==2.32.5
+Requires-Dist: tqdm==4.67.1
+Requires-Dist: openai==2.5.0
+Requires-Dist: python-dotenv==1.1.1
+Requires-Dist: platformdirs==4.5.0
+Provides-Extra: test
+Requires-Dist: pytest==8.4.2; extra == "test"
+Requires-Dist: pytest-asyncio>=1.2.0; extra == "test"
+Requires-Dist: colorama==0.4.6; extra == "test"
+Dynamic: license-file
+
+
+
+# llama-cpp-py
+
+Python wrapper for running the [llama.cpp](https://github.com/ggml-org/llama.cpp) server with automatic or manual binary management.  
+Runs the server in a separate subprocess supporting both synchronous and asynchronous APIs.
+
+
+## Requirements
+
+Python 3.10 or higher.
+
+
+## Installation
+
+Install from PyPI
+```sh
+pip install llama-cpp-py
+```
+
+Install from source
+```sh
+git clone https://github.com/sergey21000/llama-cpp-py
+cd llama-cpp-py
+pip install -e .
+```
+
+
+## Quick Start
+
+More examples in the Google Colab notebook <a href="https://colab.research.google.com/drive/17f6tD5TM9EP52-3NZtZ1qQ-QrrLUTBEG"><img src="https://img.shields.io/static/v1?message=Open%20in%20Colab&logo=googlecolab&labelColor=5c5c5c&color=0f80c1&label=%20" alt="Open in Colab"></a>
+
+
+### 1. Set up environment file for llama.cpp
+
+Creating an `env.llama` file with variables for llama.cpp server
+```sh
+# download example env file
+wget https://github.com/sergey21000/llama-cpp-py/raw/main/env.llama
+# or create manually
+nano env.llama
+```
+
+Example `env.llama` content:
+```env
+# llama.cpp server environment variables
+# See: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#usage
+
+# Model source
+LLAMA_ARG_HF_REPO=bartowski/Qwen_Qwen3-4B-GGUF
+LLAMA_ARG_HF_FILE=Qwen_Qwen3-4B-Q4_K_M.gguf
+
+# Alternative model options
+# LLAMA_ARG_MODEL_URL=https://huggingface.co/bartowski/Qwen_Qwen3-4B-GGUF/resolve/main/Qwen_Qwen3-4B-Q4_K_M.gguf
+# LLAMA_ARG_MODEL=gguf_models/Qwen_Qwen3-4B-Q4_K_M.gguf
+# LLAMA_ARG_MODEL=D:/models/Qwen_Qwen3-4B-Q4_K_M.gguf
+
+# Server configuration
+LLAMA_ARG_JINJA=1
+LLAMA_ARG_CTX_SIZE=4096
+LLAMA_ARG_NO_WEBUI=1
+LLAMA_ARG_N_PARALLEL=1
+LLAMA_ARG_N_GPU_LAYERS=-1
+
+# Network endpoint
+LLAMA_ARG_PORT=8080
+LLAMA_ARG_HOST=127.0.0.1
+```
+
+### 2. Launch the server and send requests
+
+Launching a synchronous server based on the latest [llama.cpp release](https://github.com/ggml-org/llama.cpp/releases) version
+```python
+import os
+from dotenv import dotenv_values
+from openai import OpenAI
+from llama_cpp_py import LlamaSyncServer
+
+
+# environment variables for llama.cpp
+llama_env = dotenv_values('env.llama')
+llama_env.update(os.environ)
+
+# auto-download last release and start server
+# set verbose=True to display server logs
+server = LlamaSyncServer()
+server.start(verbose=False, env=llama_env)
+
+
+# sending requests with OpenAI client
+client = OpenAI(
+	base_url=server.server_url + '/v1',
+	api_key='sk-no-key-required',
+)
+response = client.chat.completions.create(
+    model='local',
+    messages=[{'role': 'user', 'content': 'Hello!'}]
+)
+
+# stopping the server
+server.stop()
+```
+
+Launching an asynchronous server based on a specific release version
+```python
+import os
+import asyncio
+from openai import AsyncOpenAI
+from dotenv import dotenv_values
+from llama_cpp_py import LlamaAsyncServer, LlamaReleaseManager
+
+
+# environment variables for llama.cpp
+llama_env = dotenv_values('env.llama')
+llama_env.update(os.environ)
+
+# a) download a release by a specific tag with the 'cuda' priority in the title
+# set tag='latest' to use the latest llama.cpp release version
+# optionally specify priority_patterns to prefer certain builds (e.g. 'cuda' or 'cpu')
+release_manager = LlamaReleaseManager(tag='b6780', priority_patterns=['cuda'])
+
+# b) or set a specific release url in zip format
+# release_manager = LlamaReleaseManager(
+#     release_zip_url='https://github.com/ggml-org/llama.cpp/releases/download/b6780/llama-b6780-bin-win-cuda-12.4-x64.zip'
+# )
+
+# c) or selecting the compiled directory llama.cpp
+# release_manager = LlamaReleaseManager(release_dir='/content/llama.cpp/build/bin')
+	
+async def main():
+    # start llama.cpp server (set verbose=True to display server logs)
+    llama_server = LlamaAsyncServer(verbose=False, release_manager=release_manager)
+    await llama_server.start(env=llama_env)
+
+    # sending requests with OpenAI client
+    client = AsyncOpenAI(
+        base_url=f'{llama_server.server_url}/v1',
+        api_key='sk-no-key-required',
+    )
+    stream_response = await client.chat.completions.create(
+        model='local',
+        messages=[{'role': 'user', 'content': 'How are you?'}],
+        stream=True,
+        temperature=0.8,
+        max_tokens=-1,
+        extra_body=dict(
+            top_k=40,
+            reasoning_format='none',
+            chat_template_kwargs=dict(
+                enable_thinking=True,
+            ),
+        ),
+    )
+    full_response = ''
+    async for chunk in stream_response:
+        if (token := chunk.choices[0].delta.content) is not None:
+            full_response += token
+            print(token, end='', flush=True)
+
+    # stopping the server
+    await llama_server.stop()
+
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+
+## Troubleshooting
+
+If the server fails to start or behaves unexpectedly, check the following:
+- Check that the model path or URL in `env.llama` is correct
+- Verify that the port is not already in use
+- Try setting `verbose=True` to see server logs
+```python
+llama_server = LlamaAsyncServer(verbose=True)
+```
+- Link to the [llama.cpp release](https://github.com/ggml-org/llama.cpp/releases) archive appropriate for your system via 
+```python
+LlamaReleaseManager(release_zip_url=url)
+```
+- Or use the path to the directory with the pre-compiled llama.cpp 
+```python
+LlamaReleaseManager(release_dir=path_to_binaries)
+```
+
+llama.cpp binary releases are downloaded to:  
+- **Windows**
+```
+%LOCALAPPDATA%\llama-cpp-py\releases
+```
+- **Linux**
+```
+~/.local/share/llama-cpp-py/releases
+```
+- **MacOS**
+```
+~/Library/Application Support/llama-cpp-py/releases
+```
+See [platformdirs examle output](https://github.com/tox-dev/platformdirs?tab=readme-ov-file#example-output)
+
+
+## License
+
+This project is licensed under the terms of the [MIT](./LICENSE) license.
+

llama_cpp_py-0.1.11/README.md

@@ -0,0 +1,205 @@
+
+
+# llama-cpp-py
+
+Python wrapper for running the [llama.cpp](https://github.com/ggml-org/llama.cpp) server with automatic or manual binary management.  
+Runs the server in a separate subprocess supporting both synchronous and asynchronous APIs.
+
+
+## Requirements
+
+Python 3.10 or higher.
+
+
+## Installation
+
+Install from PyPI
+```sh
+pip install llama-cpp-py
+```
+
+Install from source
+```sh
+git clone https://github.com/sergey21000/llama-cpp-py
+cd llama-cpp-py
+pip install -e .
+```
+
+
+## Quick Start
+
+More examples in the Google Colab notebook <a href="https://colab.research.google.com/drive/17f6tD5TM9EP52-3NZtZ1qQ-QrrLUTBEG"><img src="https://img.shields.io/static/v1?message=Open%20in%20Colab&logo=googlecolab&labelColor=5c5c5c&color=0f80c1&label=%20" alt="Open in Colab"></a>
+
+
+### 1. Set up environment file for llama.cpp
+
+Creating an `env.llama` file with variables for llama.cpp server
+```sh
+# download example env file
+wget https://github.com/sergey21000/llama-cpp-py/raw/main/env.llama
+# or create manually
+nano env.llama
+```
+
+Example `env.llama` content:
+```env
+# llama.cpp server environment variables
+# See: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#usage
+
+# Model source
+LLAMA_ARG_HF_REPO=bartowski/Qwen_Qwen3-4B-GGUF
+LLAMA_ARG_HF_FILE=Qwen_Qwen3-4B-Q4_K_M.gguf
+
+# Alternative model options
+# LLAMA_ARG_MODEL_URL=https://huggingface.co/bartowski/Qwen_Qwen3-4B-GGUF/resolve/main/Qwen_Qwen3-4B-Q4_K_M.gguf
+# LLAMA_ARG_MODEL=gguf_models/Qwen_Qwen3-4B-Q4_K_M.gguf
+# LLAMA_ARG_MODEL=D:/models/Qwen_Qwen3-4B-Q4_K_M.gguf
+
+# Server configuration
+LLAMA_ARG_JINJA=1
+LLAMA_ARG_CTX_SIZE=4096
+LLAMA_ARG_NO_WEBUI=1
+LLAMA_ARG_N_PARALLEL=1
+LLAMA_ARG_N_GPU_LAYERS=-1
+
+# Network endpoint
+LLAMA_ARG_PORT=8080
+LLAMA_ARG_HOST=127.0.0.1
+```
+
+### 2. Launch the server and send requests
+
+Launching a synchronous server based on the latest [llama.cpp release](https://github.com/ggml-org/llama.cpp/releases) version
+```python
+import os
+from dotenv import dotenv_values
+from openai import OpenAI
+from llama_cpp_py import LlamaSyncServer
+
+
+# environment variables for llama.cpp
+llama_env = dotenv_values('env.llama')
+llama_env.update(os.environ)
+
+# auto-download last release and start server
+# set verbose=True to display server logs
+server = LlamaSyncServer()
+server.start(verbose=False, env=llama_env)
+
+
+# sending requests with OpenAI client
+client = OpenAI(
+	base_url=server.server_url + '/v1',
+	api_key='sk-no-key-required',
+)
+response = client.chat.completions.create(
+    model='local',
+    messages=[{'role': 'user', 'content': 'Hello!'}]
+)
+
+# stopping the server
+server.stop()
+```
+
+Launching an asynchronous server based on a specific release version
+```python
+import os
+import asyncio
+from openai import AsyncOpenAI
+from dotenv import dotenv_values
+from llama_cpp_py import LlamaAsyncServer, LlamaReleaseManager
+
+
+# environment variables for llama.cpp
+llama_env = dotenv_values('env.llama')
+llama_env.update(os.environ)
+
+# a) download a release by a specific tag with the 'cuda' priority in the title
+# set tag='latest' to use the latest llama.cpp release version
+# optionally specify priority_patterns to prefer certain builds (e.g. 'cuda' or 'cpu')
+release_manager = LlamaReleaseManager(tag='b6780', priority_patterns=['cuda'])
+
+# b) or set a specific release url in zip format
+# release_manager = LlamaReleaseManager(
+#     release_zip_url='https://github.com/ggml-org/llama.cpp/releases/download/b6780/llama-b6780-bin-win-cuda-12.4-x64.zip'
+# )
+
+# c) or selecting the compiled directory llama.cpp
+# release_manager = LlamaReleaseManager(release_dir='/content/llama.cpp/build/bin')
+	
+async def main():
+    # start llama.cpp server (set verbose=True to display server logs)
+    llama_server = LlamaAsyncServer(verbose=False, release_manager=release_manager)
+    await llama_server.start(env=llama_env)
+
+    # sending requests with OpenAI client
+    client = AsyncOpenAI(
+        base_url=f'{llama_server.server_url}/v1',
+        api_key='sk-no-key-required',
+    )
+    stream_response = await client.chat.completions.create(
+        model='local',
+        messages=[{'role': 'user', 'content': 'How are you?'}],
+        stream=True,
+        temperature=0.8,
+        max_tokens=-1,
+        extra_body=dict(
+            top_k=40,
+            reasoning_format='none',
+            chat_template_kwargs=dict(
+                enable_thinking=True,
+            ),
+        ),
+    )
+    full_response = ''
+    async for chunk in stream_response:
+        if (token := chunk.choices[0].delta.content) is not None:
+            full_response += token
+            print(token, end='', flush=True)
+
+    # stopping the server
+    await llama_server.stop()
+
+if __name__ == '__main__':
+    asyncio.run(main())
+```
+
+
+## Troubleshooting
+
+If the server fails to start or behaves unexpectedly, check the following:
+- Check that the model path or URL in `env.llama` is correct
+- Verify that the port is not already in use
+- Try setting `verbose=True` to see server logs
+```python
+llama_server = LlamaAsyncServer(verbose=True)
+```
+- Link to the [llama.cpp release](https://github.com/ggml-org/llama.cpp/releases) archive appropriate for your system via 
+```python
+LlamaReleaseManager(release_zip_url=url)
+```
+- Or use the path to the directory with the pre-compiled llama.cpp 
+```python
+LlamaReleaseManager(release_dir=path_to_binaries)
+```
+
+llama.cpp binary releases are downloaded to:  
+- **Windows**
+```
+%LOCALAPPDATA%\llama-cpp-py\releases
+```
+- **Linux**
+```
+~/.local/share/llama-cpp-py/releases
+```
+- **MacOS**
+```
+~/Library/Application Support/llama-cpp-py/releases
+```
+See [platformdirs examle output](https://github.com/tox-dev/platformdirs?tab=readme-ov-file#example-output)
+
+
+## License
+
+This project is licensed under the terms of the [MIT](./LICENSE) license.
+

llama_cpp_py-0.1.11/llama_cpp_py/__init__.py

@@ -0,0 +1,5 @@
+from llama_cpp_py.server.sync import LlamaSyncServer
+from llama_cpp_py.server.async_ import LlamaAsyncServer
+from llama_cpp_py.release_manager.manager import LlamaReleaseManager
+
+__all__ = ['LlamaSyncServer', 'LlamaAsyncServer', 'LlamaReleaseManager']

llama_cpp_py-0.1.11/llama_cpp_py/logger.py

@@ -0,0 +1,4 @@
+import logging
+
+logger = logging.getLogger('llama-cpp-py')
+logger.addHandler(logging.NullHandler())

llama_cpp_py-0.1.11/llama_cpp_py/release_manager/base.py

@@ -0,0 +1,209 @@
+import platform
+import zipfile
+from pathlib import Path
+
+import requests
+from tqdm import tqdm
+
+from llama_cpp_py.logger import logger
+
+
+class GithubReleaseManager:
+    def __init__(
+        self,
+        releases_api_url: str,
+        releases_dir: str | Path,
+        tag: str = 'latest',
+        release_zip_url: str = '',
+        exclude_patterns: list[str] | None = None,
+        priority_patterns: list[str] | None = None,
+    ):
+        self.validate_releases_api_url(releases_api_url)
+        self.releases_api_url = releases_api_url
+        self.releases_dir = Path(releases_dir)
+        if release_zip_url:
+            tag = self.get_tag_name_from_url(release_zip_url)
+        elif tag == 'latest':
+            tag = self.get_tag_name_from_url(self.releases_api_url + '/latest')
+        self.tag = tag
+        if not release_zip_url:
+            release_zip_url = self.get_release_zip_url(
+                tag=self.tag,
+                exclude_patterns=exclude_patterns,
+                priority_patterns=priority_patterns,
+            )
+        self.validate_release_zip_url(release_zip_url)
+        self.release_dir = self.releases_dir / Path(release_zip_url).stem
+        if not self.release_dir.exists():
+            self.download_and_extract_zip(
+            zip_url=release_zip_url,
+            extract_dir=self.release_dir,
+        )
+
+    @staticmethod
+    def validate_releases_api_url(releases_api_url):
+        # https://api.github.com/repos/ggml-org/llama.cpp/releases
+        if not (
+            releases_api_url.startswith('https://api.github.com/repos/')
+            and releases_api_url.endswith('/releases')
+        ):
+            raise ValueError(
+                'The URL with releases must start with '
+                'https://api.github.com/repos/ and end with /releases)'
+            )
+
+    @staticmethod
+    def validate_release_zip_url(release_zip_url: str) -> None:
+        # https://github.com/ggml-org/llama.cpp/releases/download/b6752/cudart-llama-bin-win-cuda-12.4-x64.zip
+        if not (
+            release_zip_url.startswith('https://github.com/')
+            and release_zip_url.endswith('.zip')
+        ):
+            raise ValueError(
+                'The URL with release must start with '
+                'https://github.com/ and end with .zip)'
+            )
+
+    @staticmethod
+    def get_tag_name_from_url(url: str) -> str:
+        if url.endswith('/releases/latest') and 'api.github.com' in url:
+            response = requests.get(url)
+            response.raise_for_status()
+            release_data = response.json()
+            if not isinstance(release_data, dict):
+                raise ValueError(
+                    f'Returned a list instead of a dictionary at requests.get("{url}").json().\n'
+                    'The URL does not lead to the page of one release'
+                )
+            tag_name = release_data.get('tag_name')
+        else:
+            tag_name = url.split('releases/download/')[-1].split('/')[0]
+        if not tag_name:
+            raise ValueError(f'Tag not found at {url}')
+        return tag_name
+
+    def get_release_zip_url(
+        self,
+        tag: str,
+        exclude_patterns: list[str] | None = None,
+        priority_patterns: list[str] | None = None,
+    ) -> str:
+        zip_assets = self.get_release_zip_assets(tag=tag)
+        zip_asset = self.get_matched_asset(
+            assets=zip_assets,
+            exclude_patterns=exclude_patterns,
+            priority_patterns=priority_patterns,
+        )
+        return zip_asset['url']
+
+    def get_release_zip_assets(self, tag: str) -> list[dict[str, str]]:
+        '''Get all links to zip archives from the specified or latest release'''
+        api_url = f'{self.releases_api_url}/tags/{tag}'
+        response = requests.get(api_url)
+        response.raise_for_status()
+        release_data = response.json()
+        zip_assets = []
+        for asset in release_data['assets']:
+            if asset['name'].endswith('.zip'):
+                zip_assets.append({
+                    'name': asset['name'],
+                    'tag_name': release_data['tag_name'],
+                    'url': asset['browser_download_url'],
+                    'size': f'{asset["size"] // 1024**2} MB',
+                })
+        return zip_assets
+
+    def get_matched_asset(
+        self,
+        assets: list[dict[str, str]],
+        exclude_patterns: list[str] | None = None,
+        priority_patterns: list[str] | None = None,
+    ) -> dict[str, str]:
+        '''Selects the appropriate archive based on the current OS and architecture'''
+        os_name, arch = self.detect_system()
+        matched_assets = []
+        for asset in assets:
+            name = asset['name'].lower()
+            if os_name not in name or arch not in name:
+                continue
+            if exclude_patterns and any(p in name for p in exclude_patterns):
+                continue
+            matched_assets.append(asset)
+        if not matched_assets:
+            raise RuntimeError(f'No suitable archive found for {os_name}-{arch}')
+        if priority_patterns:
+            for pattern in priority_patterns:
+                for asset in matched_assets:
+                    name = asset['name'].lower()
+                    if pattern in name:
+                        matched_assets = [asset]
+                        break
+                if len(matched_assets) == 1:
+                    break
+        if len(matched_assets) > 1:
+            logger.warning(
+                f'More than one archive match found, the first one will be selected:'
+                f'{[d.get("name") for d in matched_assets]}'
+            )
+        return matched_assets[0]
+
+    @staticmethod
+    def detect_system() -> tuple[str, str]:
+        '''Determines the current platform (OS + architecture)'''
+        os_name = platform.system().lower()
+        arch = platform.machine().lower()
+        if os_name == 'windows':
+            os_name = 'win'
+        elif os_name == 'linux':
+            os_name = 'ubuntu'
+        elif os_name == 'darwin':
+            os_name = 'macos'
+        if arch in ('x86_64', 'amd64'):
+            arch = 'x64'
+        elif arch in ('arm64', 'aarch64'):
+            arch = 'arm64'
+        return os_name, arch
+
+    @staticmethod
+    def download_file(file_url: str, file_path: str | Path) -> None:
+        response = requests.get(file_url, stream=True)
+        if response.status_code != 200:
+            raise Exception(
+                f'The file is not available for download at the link: {file_url}'
+            )
+        total_size = int(response.headers.get('content-length', 0))
+        progress_tqdm = tqdm(
+            desc=f'Downoading release: {Path(file_path).name}',
+            total=total_size,
+            unit='iB',
+            unit_scale=True,
+        )
+        with open(file_path, 'wb') as file:
+            for data in response.iter_content(chunk_size=4096):
+                size = file.write(data)
+                progress_tqdm.update(size)
+        progress_tqdm.close()
+
+    @staticmethod
+    def extract_zip(zip_path: Path, extract_dir: Path) -> None:
+        with zipfile.ZipFile(zip_path, 'r') as archive:
+            archive.extractall(path=extract_dir)
+
+    @classmethod
+    def download_and_extract_zip(
+        cls,
+        zip_url: str,
+        extract_dir: Path,
+        override: bool = False,
+        set_execute_permissions: bool = True,
+    ) -> None:
+        extract_dir.mkdir(exist_ok=True, parents=True)
+        zip_path = extract_dir / Path(zip_url).name
+        logger.info(f'Loading file {zip_url} to path {zip_path}')
+        cls.download_file(file_url=zip_url, file_path=zip_path)
+        cls.extract_zip(zip_path=zip_path, extract_dir=extract_dir)
+        zip_path.unlink(missing_ok=True)
+        if set_execute_permissions and platform.system() != 'Windows':
+            for file in extract_dir.rglob('*'):
+                if file.is_file():
+                    file.chmod(0o755)