moxing 0.1.0__py3-none-any.whl

moxing-0.1.0.dist-info/METADATA

@@ -0,0 +1,233 @@
+Metadata-Version: 2.4
+Name: moxing
+Version: 0.1.0
+Summary: Python wrapper for llama.cpp - OpenAI API compatible LLM backend with auto GPU detection
+Author: llama.cpp community
+License: MIT
+Project-URL: Homepage, https://github.com/ggml-org/llama.cpp
+Project-URL: Documentation, https://github.com/ggml-org/llama.cpp/tree/master/tools/server
+Project-URL: Repository, https://github.com/ggml-org/llama.cpp
+Project-URL: Issues, https://github.com/ggml-org/llama.cpp/issues
+Keywords: llama,llama.cpp,gguf,openai,api,gpu,vulkan,cuda,ai,llm
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0
+Requires-Dist: psutil>=5.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == "openai"
+Provides-Extra: hf
+Requires-Dist: huggingface_hub>=0.20.0; extra == "hf"
+Provides-Extra: modelscope
+Requires-Dist: modelscope>=1.10.0; extra == "modelscope"
+Provides-Extra: all
+Requires-Dist: openai>=1.0.0; extra == "all"
+Requires-Dist: huggingface_hub>=0.20.0; extra == "all"
+Requires-Dist: modelscope>=1.10.0; extra == "all"
+
+# moxing (模型)
+
+Python wrapper for llama.cpp - OpenAI API compatible LLM backend with automatic GPU detection and model downloading.
+
+**moxing** (模型) means "model" in Chinese. A simple, unified interface for running LLMs locally.
+
+## Features
+
+- **Auto GPU Detection**: Automatically detects and configures the best GPU backend (Vulkan, CUDA, ROCm, Metal)
+- **Model Downloading**: Download GGUF models from HuggingFace and ModelScope
+- **OpenAI API Compatible**: Drop-in replacement for OpenAI API
+- **Function Calling**: Support for tools and function calling
+- **Pre-built Binaries**: Automatically downloads pre-built llama.cpp binaries
+- **Benchmark**: Measure tokens/second performance like ollama
+
+## Installation
+
+```bash
+pip install moxing
+```
+
+## Quick Start
+
+### Command Line
+
+```bash
+# List available GPUs
+pyllm devices
+
+# Run inference with a model
+pyllm run ./model.gguf -p "Hello, world!"
+
+# Quick speed test
+pyllm speed ./model.gguf
+
+# Benchmark performance
+pyllm bench ./model.gguf
+
+# Download a model
+pyllm download llama-3.2-3b -q Q4_K_M
+
+# Start an OpenAI-compatible server
+pyllm serve llama-3.2-3b -p 8080
+```
+
+### Python API
+
+```python
+from pyllm import quick_run, quick_server, Client
+
+# Quick inference
+result = quick_run("llama-3.2-3b", "Write a haiku about coding")
+print(result)
+
+# Start server with auto-configuration
+with quick_server("llama-3.2-3b") as server:
+    client = Client(server.base_url)
+    
+    response = client.chat.completions.create(
+        model="llama",
+        messages=[{"role": "user", "content": "Hello!"}]
+    )
+    print(response.choices[0]["message"]["content"])
+```
+
+### Auto GPU Detection
+
+```python
+from pyllm import DeviceDetector, AutoRunner
+
+# Detect available GPUs
+detector = DeviceDetector()
+devices = detector.detect()
+for device in devices:
+    print(f"{device.name} ({device.backend.value}, {device.memory_gb:.1f}GB)")
+
+# Get optimal configuration
+config = detector.get_best_device(model_size_gb=5.0)
+print(f"Best device: {config.device.name}")
+print(f"Recommended GPU layers: {config.n_gpu_layers}")
+```
+
+### Model Download
+
+```python
+from pyllm import ModelDownloader
+
+downloader = ModelDownloader()
+
+# Download from HuggingFace
+path = downloader.download("Qwen/Qwen2.5-7B-Instruct-GGUF", "Q4_K_M.gguf")
+
+# Download from ModelScope
+path = downloader.download(
+    "LLM-Research/Meta-Llama-3-8B-Instruct-GGUF",
+    "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
+    source="modelscope"
+)
+```
+
+## CLI Commands
+
+| Command | Description |
+|---------|-------------|
+| `pyllm serve` | Start OpenAI-compatible server |
+| `pyllm run` | Run inference with a model |
+| `pyllm chat` | Interactive chat with a model |
+| `pyllm bench` | Benchmark model performance |
+| `pyllm speed` | Quick speed test |
+| `pyllm info` | Show model info and estimates |
+| `pyllm download` | Download a model |
+| `pyllm models` | List available models |
+| `pyllm devices` | List GPU devices |
+| `pyllm config` | Show optimal configuration |
+| `pyllm diagnose` | Diagnose system setup |
+
+## Popular Models
+
+| Name | Description | Sizes |
+|------|-------------|-------|
+| llama-3.2-3b | Llama 3.2 3B | Q4_K_M, Q5_K_M, Q8_0 |
+| llama-3.1-8b | Llama 3.1 8B | Q4_K_M, Q5_K_M, Q8_0 |
+| qwen2.5-7b | Qwen 2.5 7B | Q4_K_M, Q5_K_M, Q8_0 |
+| gemma-2-9b | Gemma 2 9B | Q4_K_M, Q5_K_M, Q8_0 |
+| mistral-7b | Mistral 7B v0.3 | Q4_K_M, Q5_K_M, Q8_0 |
+| phi-3.5-mini | Phi 3.5 Mini | Q4_K_M, Q5_K_M, Q8_0 |
+| deepseek-coder-6.7b | DeepSeek Coder | Q4_K_M, Q5_K_M, Q8_0 |
+
+## GPU Backends
+
+| Backend | Platforms | Description |
+|---------|-----------|-------------|
+| Vulkan | Windows, Linux | Cross-platform GPU API, works on AMD, Intel, NVIDIA |
+| CUDA | Windows, Linux | NVIDIA GPUs |
+| ROCm | Linux | AMD GPUs |
+| Metal | macOS | Apple Silicon |
+| CPU | All | Fallback, no GPU required |
+
+## Function Calling
+
+```python
+from pyllm import Client, LlamaServer
+
+tools = [{
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get weather for a location",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {"type": "string"}
+            }
+        }
+    }
+}]
+
+with LlamaServer("model.gguf") as server:
+    client = Client(server.base_url)
+    
+    response = client.chat.completions.create(
+        model="llama",
+        messages=[{"role": "user", "content": "What's the weather in Tokyo?"}],
+        tools=tools
+    )
+    
+    if response.choices[0]["message"].get("tool_calls"):
+        print("Model wants to call:", response.choices[0]["message"]["tool_calls"])
+```
+
+## Requirements
+
+- Python 3.8+
+- Vulkan SDK (for Vulkan backend)
+- CUDA Toolkit (for CUDA backend)
+- ROCm (for ROCm backend)
+
+## License
+
+MIT License - same as llama.cpp
+
+## Links
+
+- [llama.cpp](https://github.com/ggml-org/llama.cpp)
+- [Documentation](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)
+- [Issues](https://github.com/ggml-org/llama.cpp/issues)

moxing-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+pyllm/__init__.py,sha256=e7olt5uAPR67AgwNc-8T20Bw5W2ODyCtQCr_HKdcnVA,1789
+pyllm/benchmark.py,sha256=ReB27S3o-olfSwOtvYU2a301c0GEllBqmU87a-IsiXU,13631
+pyllm/binaries.py,sha256=3fCYT382qDHj2-MhY2mZb4iFLR8UfjssCzWuf2oOzqc,11939
+pyllm/cli.py,sha256=BmYxokxqK-6NSicKPyjaJZdk2nLHp_selSCcjC0dumo,25153
+pyllm/client.py,sha256=f_zJH-mGjukd3U76ntSRABgeLdd_dduK43m-jkHWJN8,5185
+pyllm/device.py,sha256=cEsfMm7DYyH5jG1LqAqoVFU4YXpjff8kMwVc3PPhwK8,10698
+pyllm/models.py,sha256=0KJE3Z9wEYBbRgFmbZMMD8N0nLCaTWNFmDvrXmiSnsw,18822
+pyllm/runner.py,sha256=emScwl-zcXtbwj7J0xe6Dyov6QJm_mKrQXZOSY1vlkY,10114
+pyllm/server.py,sha256=D6gl9Z9APCBLataG9tShOgdnjcT5QCopYMULjb2EZEI,8071
+pyllm/bin/darwin/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyllm/bin/linux/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyllm/bin/windows/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+moxing-0.1.0.dist-info/METADATA,sha256=LO46cwLrH2hEUKlE8GuWjPWyLnGxfP2ChHMJ3SIPYwA,7230
+moxing-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+moxing-0.1.0.dist-info/entry_points.txt,sha256=tk3NDRLksFK2lchSqSEgJ6Cby4kBVlNBOjYH5BGW5vg,41
+moxing-0.1.0.dist-info/top_level.txt,sha256=tdBgv9nG64-rOMYgijsbM_MtDiFIest5VKrSV3PZhoI,6
+moxing-0.1.0.dist-info/RECORD,,

moxing-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

moxing-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+moxing = pyllm.cli:app

moxing-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pyllm

pyllm/__init__.py

@@ -0,0 +1,80 @@
+"""
+pyllm - Python wrapper for llama.cpp server
+
+Provides an OpenAI API compatible interface for running GGUF models
+with GPU acceleration (Vulkan, CUDA, ROCm, Metal).
+
+Features:
+- Auto-detect best GPU device and backend
+- Download models from HuggingFace and ModelScope
+- OpenAI API compatible server
+- Function calling / tool support
+- Multimodal support
+
+Quick start:
+    from pyllm import quick_run, quick_server
+    
+    # Quick inference
+    result = quick_run("llama-3.2-3b", "Write a haiku")
+    
+    # Start server
+    with quick_server("llama-3.2-3b") as server:
+        # Use OpenAI API at http://localhost:8080/v1
+        pass
+"""
+
+__version__ = "0.1.0"
+
+from pyllm.client import Client, ChatCompletion, Message
+from pyllm.server import LlamaServer, ServerConfig, GPUInfo
+from pyllm.device import (
+    Device, DeviceConfig, DeviceDetector, BackendType,
+    detect_best_backend, get_device_config
+)
+from pyllm.models import (
+    ModelDownloader, ModelInfo, ModelRegistry, download_model
+)
+from pyllm.runner import (
+    AutoRunner, RunConfig, quick_run, quick_server
+)
+from pyllm.binaries import (
+    BinaryManager, get_binary_manager, ensure_binaries, get_server_binary
+)
+
+__all__ = [
+    # Client
+    "Client",
+    "ChatCompletion",
+    "Message",
+    
+    # Server
+    "LlamaServer",
+    "ServerConfig",
+    "GPUInfo",
+    
+    # Device
+    "Device",
+    "DeviceConfig", 
+    "DeviceDetector",
+    "BackendType",
+    "detect_best_backend",
+    "get_device_config",
+    
+    # Models
+    "ModelDownloader",
+    "ModelInfo",
+    "ModelRegistry",
+    "download_model",
+    
+    # Runner
+    "AutoRunner",
+    "RunConfig",
+    "quick_run",
+    "quick_server",
+    
+    # Binaries
+    "BinaryManager",
+    "get_binary_manager",
+    "ensure_binaries",
+    "get_server_binary",
+]