openrun-llm 1.0.0__tar.gz

openrun_llm-1.0.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: openrun-llm
+Version: 1.0.0
+Summary: Turn any local AI model into an OpenAI-compatible API
+Requires-Python: >=3.9
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: pydantic
+Requires-Dist: transformers
+Requires-Dist: torch
+Requires-Dist: accelerate

openrun_llm-1.0.0/README.md

@@ -0,0 +1,226 @@
+# 🚀 OpenRun v1.0
+
+> Run any Python AI model as an OpenAI-compatible API — instantly.
+
+---
+
+## 💡 What is OpenRun?
+
+**OpenRun** lets you take *any Python AI model code* and expose it as a fully functional **OpenAI-compatible API** with **one line of code**.
+
+No UI. No frontend. No setup headaches.
+
+Just your model → API.
+
+---
+
+## ⚡ Features
+
+* 🧠 Works with **any Python model code**
+* 🔌 OpenAI-compatible API (`/v1/chat/completions`)
+* 🔄 Full **messages support** (chat history, roles)
+* ⚡ **Streaming support** (`stream=True`)
+* 🌍 Public URL via tunnel (`--public`)
+* 🔐 Auto API key generation
+* 🧪 Works in **Google Colab, Jupyter, local**
+* 🧩 Supports HuggingFace + custom models
+
+---
+
+## 🚀 Quick Start (1 Cell)
+
+```python
+from openrun import serve
+
+def chat(messages):
+    return "Hello from OpenRun!"
+
+serve(fn=chat, public=True)
+```
+
+---
+
+## 🌍 Output
+
+```text
+🚀 OpenRun running
+🔐 API Key: sk-or-xxxx
+
+🌍 Public URL: https://xxxx.trycloudflare.com
+📡 Endpoint: /v1/chat/completions
+```
+
+---
+
+## 🔌 Use with OpenAI SDK
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="https://your-url/v1",
+    api_key="sk-or-xxxx"
+)
+
+response = client.chat.completions.create(
+    model="openrun",
+    messages=[{"role": "user", "content": "Hello"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+---
+
+## ⚡ Streaming Example
+
+```python
+response = client.chat.completions.create(
+    model="openrun",
+    messages=[{"role": "user", "content": "Hello"}],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk.choices[0].delta.content, end="")
+```
+
+---
+
+## 🧠 Using Your Own Model
+
+Works with ANY code:
+
+```python
+def chat(messages):
+    # your model logic
+    return "Custom response"
+```
+
+OR
+
+```python
+def generate(prompt):
+    return "Simple model"
+```
+
+OpenRun auto-detects and adapts.
+
+---
+
+## 🖥 CLI Usage
+
+```bash
+openrun serve my_model.py --public
+```
+
+---
+
+## 🔐 API Key
+
+* Auto-generated if not provided
+* Or set manually:
+
+```bash
+openrun serve my_model.py --api-key mykey
+```
+
+---
+
+## 📡 API Format
+
+### Endpoint:
+
+```
+POST /v1/chat/completions
+```
+
+### Request:
+
+```json
+{
+  "messages": [
+    {"role": "user", "content": "Hello"}
+  ],
+  "stream": false
+}
+```
+
+---
+
+## 🧠 How It Works
+
+1. Loads your Python function
+2. Wraps it as an API
+3. Adds OpenAI compatibility
+4. Optionally exposes it publicly
+
+---
+
+## 🎯 Use Cases
+
+* Run LLMs in Colab as APIs
+* Share models instantly
+* Build custom AI backends
+* Replace OpenAI API locally
+
+---
+
+## 🔥 Why OpenRun?
+
+Because running models should be this simple:
+
+```python
+serve(fn=chat)
+```
+
+---
+
+## 🚀 Installation
+
+```bash
+pip install openrun
+```
+
+---
+
+## 🛠 Development
+
+```bash
+git clone https://github.com/<your-username>/openrun.git
+cd openrun
+pip install -e .
+```
+
+---
+
+## 📦 Version
+
+**v1.0 — Initial Release**
+
+---
+
+## 🤝 Contributing
+
+PRs welcome! Feel free to improve adapters, streaming, or integrations.
+
+---
+
+## ⭐ Support
+
+If you like this project, give it a ⭐ on GitHub!
+
+---
+
+## 🚀 Future Plans
+
+* Real token streaming (HF streamer)
+* Multi-model support
+* Dashboard UI
+* Cloud deployment
+
+---
+
+## ⚡ One Line Summary
+
+> Turn any Python AI model into a public OpenAI API instantly.

openrun_llm-1.0.0/adapters/__init__.py

@@ -0,0 +1 @@
+"""Adapters Module"""

openrun_llm-1.0.0/adapters/base.py

@@ -0,0 +1,9 @@
+class BaseAdapter:
+    def load(self):
+        pass
+
+    def generate(self, input_data: list) -> str:
+        pass
+
+    def stream(self, input_data: list):
+        raise NotImplementedError

openrun_llm-1.0.0/adapters/custom.py

@@ -0,0 +1,65 @@
+import importlib.util
+import sys
+import os
+import inspect
+from adapters.base import BaseAdapter
+
+class CustomAdapter(BaseAdapter):
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.custom_module = None
+        self.target_func = None
+        self.func_type = "prompt"
+
+    def load(self):
+        print(f"Loading custom model from '{self.file_path}'...")
+        if not os.path.exists(self.file_path):
+            raise FileNotFoundError(f"Custom model file not found: {self.file_path}")
+        
+        print("⚠️ Running custom code. Ensure it is trusted.")
+        spec = importlib.util.spec_from_file_location("custom_model", self.file_path)
+        self.custom_module = importlib.util.module_from_spec(spec)
+        sys.modules["custom_model"] = self.custom_module
+        spec.loader.exec_module(self.custom_module)
+        
+        if hasattr(self.custom_module, "generate"):
+            self.target_func = self.custom_module.generate
+        elif hasattr(self.custom_module, "chat"):
+            self.target_func = self.custom_module.chat
+            self.func_type = "messages"
+        elif hasattr(self.custom_module, "predict"):
+            self.target_func = self.custom_module.predict
+        else:
+            for name, obj in inspect.getmembers(self.custom_module, inspect.isfunction):
+                try:
+                    sig = inspect.signature(obj)
+                    if len(sig.parameters) == 1:
+                        self.target_func = obj
+                        break
+                except ValueError:
+                    pass
+
+        if not self.target_func:
+            print("❌ No valid function found.")
+            print("Define one of:")
+            print("- generate(prompt)")
+            print("- chat(messages)")
+            print("- predict(prompt)")
+            raise AttributeError("No valid function found in custom model.")
+
+        print(f"🧠 Using function: {self.target_func.__name__}()")
+
+    def generate(self, input_data: list) -> str:
+        if not self.target_func:
+            raise RuntimeError("Custom model not loaded. Call load() first.")
+        
+        if self.func_type == "messages":
+            return self.target_func(input_data)
+        else:
+            prompt = input_data[-1]["content"] if input_data else ""
+            return self.target_func(prompt)
+
+    def stream(self, input_data: list):
+        response = self.generate(input_data)
+        for word in response.split():
+            yield word + " "

openrun_llm-1.0.0/adapters/huggingface.py

@@ -0,0 +1,39 @@
+from adapters.base import BaseAdapter
+try:
+    from transformers import pipeline
+except ImportError:
+    raise ImportError("Please install transformers: pip install transformers torch")
+
+class HuggingFaceAdapter(BaseAdapter):
+    def __init__(self, model_name: str):
+        self.model_name = model_name
+        self.generator = None
+
+    def load(self):
+        print(f"Loading HuggingFace model '{self.model_name}'...")
+        self.generator = pipeline('text-generation', model=self.model_name, device_map="auto")
+
+    def generate(self, input_data: list) -> str:
+        if not self.generator:
+            raise RuntimeError("Model not loaded. Call load() first.")
+        
+        prompt = ""
+        if input_data:
+            for msg in input_data:
+                prompt += f"<|{msg['role']}|>\n{msg['content']}\n"
+        prompt += "<|assistant|>\n"
+        
+        # Basic generation. Adjust max_new_tokens as needed.
+        result = self.generator(prompt, max_new_tokens=200, num_return_sequences=1)
+        generated_text = result[0]['generated_text']
+        
+        # Safely remove prompt if present
+        if prompt in generated_text:
+            generated_text = generated_text.split(prompt, 1)[-1].strip()
+        
+        return generated_text
+
+    def stream(self, input_data: list):
+        response = self.generate(input_data)
+        for word in response.split():
+            yield word + " "

openrun_llm-1.0.0/api/__init__.py

@@ -0,0 +1 @@
+"""API Module"""

openrun_llm-1.0.0/api/dependencies.py

@@ -0,0 +1,31 @@
+from fastapi import Request, HTTPException, status
+from core.state import get_global_state
+
+async def verify_api_key(request: Request):
+    state = get_global_state()
+    expected_key = getattr(state.config, "api_key", None)
+    
+    # If no key is set in config, allow access
+    if not expected_key:
+        return
+        
+    error_detail = {
+        "error": {
+            "message": "Unauthorized",
+            "type": "authentication_error"
+        }
+    }
+        
+    auth_header = request.headers.get("Authorization") or request.headers.get("authorization")
+    if not auth_header:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=error_detail
+        )
+        
+    scheme, _, token = auth_header.partition(" ")
+    if scheme.lower() != "bearer" or token.strip() != expected_key:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=error_detail
+        )

openrun_llm-1.0.0/api/routes.py

@@ -0,0 +1,51 @@
+from fastapi import APIRouter, Depends
+from fastapi.responses import StreamingResponse
+from api.schemas import ChatRequest
+from core.state import get_global_state
+from api.dependencies import verify_api_key
+from model.inference import generate_response, stream_response
+import time
+import uuid
+
+router = APIRouter()
+
+@router.post("/v1/chat/completions", dependencies=[Depends(verify_api_key)])
+async def chat_completions(request: ChatRequest):
+    state = get_global_state()
+    model_name = state.config.model if state.config and state.config.model else request.model
+    model_name = model_name or "openrun"
+    
+    # Extract messages directly
+    messages = [{"role": msg.role, "content": msg.content} for msg in request.messages]
+
+    if request.stream:
+        return StreamingResponse(
+            stream_response(messages),
+            media_type="text/event-stream",
+            headers={"Cache-Control": "no-cache"}
+        )
+
+    # Call inference layer
+    response_text = generate_response(messages)
+    
+    return {
+        "id": f"chatcmpl-{uuid.uuid4().hex[:12]}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": model_name,
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": response_text
+                },
+                "finish_reason": "stop"
+            }
+        ],
+        "usage": {
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "total_tokens": 0
+        }
+    }

openrun_llm-1.0.0/api/schemas.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import List, Optional
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ChatRequest(BaseModel):
+    model: str
+    messages: List[Message]
+    stream: Optional[bool] = False

openrun_llm-1.0.0/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI Module"""

openrun_llm-1.0.0/cli/main.py

@@ -0,0 +1,26 @@
+import argparse
+import sys
+from cli.serve import run_serve
+
+def main():
+    parser = argparse.ArgumentParser(description="OpenRun - Target any local AI model via an OpenAI-compatible API")
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Serve command
+    serve_parser = subparsers.add_parser("serve", help="Start the OpenAI-compatible server")
+    serve_parser.add_argument("--model", type=str, help="Name of the model to load")
+    serve_parser.add_argument("--file", type=str, help="Path to custom model file")
+    serve_parser.add_argument("--port", type=int, default=8000, help="Port to run the server on")
+    serve_parser.add_argument("--public", action="store_true", help="Expose server publicly via Cloudflare")
+    serve_parser.add_argument("--api-key", type=str, help="Require API key for requests")
+
+    args = parser.parse_args()
+
+    if args.command == "serve":
+        run_serve(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

openrun_llm-1.0.0/cli/serve.py

@@ -0,0 +1,38 @@
+import uvicorn
+from core.config import Config
+from core.state import set_global_state
+from network.server import create_app
+from network.tunnel import start_tunnel
+from model.loader import load_model
+
+def run_serve(args):
+    print("--- OpenRun Serve ---")
+    print(f"Model  : {args.model}")
+    print(f"File   : {args.file}")
+    print(f"Port   : {args.port}")
+    print(f"Public : {args.public}")
+    print(f"API Key: {args.api_key}")
+
+    # Initialize config and state
+    config = Config(
+        model=args.model,
+        file=args.file,
+        port=args.port,
+        public=args.public,
+        api_key=args.api_key
+    )
+    set_global_state(config=config, model=None)
+
+    # Load model via loader
+    load_model(config)
+
+    # Start Cloudflare tunnel if requested
+    if config.public:
+        import time
+        time.sleep(2)
+        start_tunnel(config.port)
+
+    # Start FastAPI server
+    app = create_app()
+    print(f"Starting server on port {args.port}...")
+    uvicorn.run(app, host="0.0.0.0", port=args.port)

openrun_llm-1.0.0/core/__init__.py

@@ -0,0 +1 @@
+"""Core Module"""

openrun_llm-1.0.0/core/config.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass
+class Config:
+    model: Optional[str] = None
+    file: Optional[str] = None
+    port: int = 8000
+    public: bool = False
+    api_key: Optional[str] = None

openrun_llm-1.0.0/core/state.py

@@ -0,0 +1,22 @@
+from typing import Any, Optional
+from core.config import Config
+
+class AppState:
+    def __init__(self):
+        self.config: Optional[Config] = None
+        self.model: Optional[Any] = None
+        self.adapter: Optional[Any] = None
+
+# Global state instance
+global_state = AppState()
+
+def set_global_state(config: Optional[Config] = None, model: Optional[Any] = None, adapter: Optional[Any] = None):
+    if config is not None:
+        global_state.config = config
+    if model is not None:
+        global_state.model = model
+    if adapter is not None:
+        global_state.adapter = adapter
+
+def get_global_state() -> AppState:
+    return global_state

openrun_llm-1.0.0/model/__init__.py

@@ -0,0 +1 @@
+"""Model Module"""

openrun_llm-1.0.0/model/inference.py

@@ -0,0 +1,36 @@
+from core.state import get_global_state
+import json
+
+def stream_response(messages: list):
+    state = get_global_state()
+
+    if not hasattr(state, "adapter") or not state.adapter:
+        yield "data: [ERROR] No model loaded\n\n"
+        return
+
+    try:
+        # Initial role chunk
+        yield f"data: {json.dumps({'id': 'chatcmpl-openrun', 'object': 'chat.completion.chunk', 'choices': [{'delta': {'role': 'assistant'}}]})}\n\n"
+
+        # Content chunks
+        for chunk in state.adapter.stream(messages):
+            yield f"data: {json.dumps({'id': 'chatcmpl-openrun', 'object': 'chat.completion.chunk', 'choices': [{'delta': {'content': chunk}}]})}\n\n"
+
+        # Finish reason chunk
+        yield f"data: {json.dumps({'id': 'chatcmpl-openrun', 'object': 'chat.completion.chunk', 'choices': [{'finish_reason': 'stop'}]})}\n\n"
+        
+        yield "data: [DONE]\n\n"
+
+    except Exception as e:
+        yield f"data: {json.dumps({'error': str(e)})}\n\n"
+
+def generate_response(messages: list) -> str:
+    state = get_global_state()
+    
+    if not hasattr(state, "adapter") or not state.adapter:
+        return "Warning: No model loaded. Please provide --model or --file."
+    
+    try:
+        return state.adapter.generate(messages)
+    except Exception as e:
+        return f"Error: Model generation failed - {str(e)}"

openrun_llm-1.0.0/model/loader.py

@@ -0,0 +1,22 @@
+from core.config import Config
+from core.state import get_global_state
+
+def load_model(config: Config):
+    adapter = None
+    
+    if config.file:
+        from adapters.custom import CustomAdapter
+        adapter = CustomAdapter(config.file)
+    elif config.model:
+        from adapters.huggingface import HuggingFaceAdapter
+        adapter = HuggingFaceAdapter(config.model)
+    else:
+        print("Warning: Neither --model nor --file specified. Running in dummy mode.")
+        return
+        
+    adapter.load()
+    
+    # Store adapter in global state
+    state = get_global_state()
+    state.adapter = adapter
+    print("Model loaded successfully.")

openrun_llm-1.0.0/network/__init__.py

@@ -0,0 +1 @@
+"""Network Module"""

openrun_llm-1.0.0/network/server.py

@@ -0,0 +1,14 @@
+from fastapi import FastAPI
+from api.routes import router as api_router
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="OpenRun API", version="0.1.0")
+
+    @app.get("/health")
+    async def health():
+        return {"status": "ok"}
+
+    # Include the OpenAI-compatible routes
+    app.include_router(api_router)
+    
+    return app

openrun_llm-1.0.0/network/tunnel.py

@@ -0,0 +1,49 @@
+import subprocess
+import threading
+import shutil
+import re
+
+def _monitor_tunnel(process):
+    url_pattern = re.compile(r"https://[a-zA-Z0-9-]+\.trycloudflare\.com")
+    found = False
+    
+    for line in process.stdout:
+        match = url_pattern.search(line)
+        if match and not found:
+            print(f"\n🌍 Public URL: {match.group(0)}\n")
+            found = True
+            # We found the URL, but we should continue reading the stream 
+            # to prevent the pipe buffer from filling up and blocking the process.
+
+    if not found:
+        print("⚠️ Could not detect public URL automatically. Check logs above.")
+        
+def start_tunnel(port: int):
+    """Starts a Cloudflare tunnel in the background pointing to the local port."""
+    if not shutil.which("cloudflared"):
+        print("\n⚠️  Error: 'cloudflared' is not installed or not in PATH.")
+        print("To use --public, please install cloudflared:")
+        print("  - Mac: brew install cloudflare/cloudflare/cloudflared")
+        print("  - Linux: wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb && dpkg -i cloudflared-linux-amd64.deb")
+        print("  - Windows: winget install cloudflared")
+        print("Or visit: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/\n")
+        return
+
+    print("🌐 Starting Cloudflare tunnel...")
+    
+    try:
+        # cloudflared logs everything to stderr, so we merge it into stdout
+        process = subprocess.Popen(
+            ["cloudflared", "tunnel", "--url", f"http://localhost:{port}"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1
+        )
+        
+        # Monitor the output in a separate daemon thread so it doesn't block
+        thread = threading.Thread(target=_monitor_tunnel, args=(process,), daemon=True)
+        thread.start()
+        
+    except Exception as e:
+        print(f"\n⚠️  Failed to start Cloudflare tunnel: {e}\n")

openrun_llm-1.0.0/openrun_llm.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: openrun-llm
+Version: 1.0.0
+Summary: Turn any local AI model into an OpenAI-compatible API
+Requires-Python: >=3.9
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: pydantic
+Requires-Dist: transformers
+Requires-Dist: torch
+Requires-Dist: accelerate

openrun_llm-1.0.0/openrun_llm.egg-info/SOURCES.txt

@@ -0,0 +1,31 @@
+README.md
+pyproject.toml
+adapters/__init__.py
+adapters/base.py
+adapters/custom.py
+adapters/huggingface.py
+api/__init__.py
+api/dependencies.py
+api/routes.py
+api/schemas.py
+cli/__init__.py
+cli/main.py
+cli/serve.py
+core/__init__.py
+core/config.py
+core/state.py
+model/__init__.py
+model/inference.py
+model/loader.py
+network/__init__.py
+network/server.py
+network/tunnel.py
+openrun_llm.egg-info/PKG-INFO
+openrun_llm.egg-info/SOURCES.txt
+openrun_llm.egg-info/dependency_links.txt
+openrun_llm.egg-info/entry_points.txt
+openrun_llm.egg-info/requires.txt
+openrun_llm.egg-info/top_level.txt
+security/__init__.py
+streaming/__init__.py
+utils/__init__.py

openrun_llm-1.0.0/openrun_llm.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

openrun_llm-1.0.0/openrun_llm.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+openrun = cli.main:main

openrun_llm-1.0.0/openrun_llm.egg-info/requires.txt

@@ -0,0 +1,6 @@
+fastapi
+uvicorn
+pydantic
+transformers
+torch
+accelerate

openrun_llm-1.0.0/openrun_llm.egg-info/top_level.txt

@@ -0,0 +1,9 @@
+adapters
+api
+cli
+core
+model
+network
+security
+streaming
+utils

openrun_llm-1.0.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openrun-llm"
+version = "1.0.0"
+description = "Turn any local AI model into an OpenAI-compatible API"
+requires-python = ">=3.9"
+dependencies = [
+    "fastapi",
+    "uvicorn",
+    "pydantic",
+    "transformers",
+    "torch",
+    "accelerate"
+]
+
+[project.scripts]
+openrun = "cli.main:main"
+
+[tool.setuptools]
+packages = [
+  "cli",
+  "api",
+  "core",
+  "model",
+  "adapters",
+  "network",
+  "security",
+  "streaming",
+  "utils"
+]

openrun_llm-1.0.0/security/__init__.py

@@ -0,0 +1 @@
+"""Security Module"""

openrun_llm-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

openrun_llm-1.0.0/streaming/__init__.py

@@ -0,0 +1 @@
+"""Streaming Module"""

openrun_llm-1.0.0/utils/__init__.py

@@ -0,0 +1 @@
+"""Utils Module"""