local-ai-runtime 0.1.0__py3-none-any.whl

backend/controllers/chat_controller.py

@@ -0,0 +1,42 @@
+"""Controller for chat request handling."""
+
+from fastapi import HTTPException
+
+from services.inference_service import generate_response, InferenceError
+from models.chat_message import ChatMessage, Role
+from services.conversation_service import get_conversation
+
+
+async def handle_chat_request(request) -> dict:
+    user_message = ChatMessage(role=Role.USER, content=request.message)
+
+    conversation_history = None
+    if request.conversation_id:
+        conv = get_conversation(request.conversation_id)
+        if conv:
+            conversation_history = [
+                ChatMessage(role=Role(m["role"]), content=m["content"])
+                for m in conv["messages"]
+            ]
+
+    try:
+        result = await generate_response(
+            message=user_message,
+            conversation_id=request.conversation_id,
+            conversation_history=conversation_history,
+        )
+    except InferenceError as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+    # Auto-save user message and assistant response to conversation
+    if request.conversation_id:
+        from services.conversation_service import add_message_to_conversation
+        add_message_to_conversation(request.conversation_id, "user", request.message)
+        add_message_to_conversation(request.conversation_id, "assistant", result.content)
+
+    return {
+        "response": result.content,
+        "model": result.model_name,
+        "tokens_used": result.tokens_used,
+        "finish_reason": result.finish_reason,
+    }

backend/controllers/model_controller.py

@@ -0,0 +1,15 @@
+"""Controller for model management request handling."""
+
+from services.model_service import scan_model_directory, get_active_config
+
+
+def list_available_models() -> dict:
+    models = scan_model_directory()
+    return {"models": models}
+
+
+def get_active_model() -> dict:
+    config = get_active_config()
+    if config is None:
+        return {"active_model": None}
+    return config.to_dict()

backend/models/__init__.py

@@ -0,0 +1,2 @@
+from models.chat_message import ChatMessage, Conversation, Role
+from models.model_config import ModelConfig, GenerationParams, CHAT_TEMPLATES

backend/models/chat_message.py

@@ -0,0 +1,80 @@
+# backend/models/chat_message.py
+#
+# Data model for a single chat message.
+#
+# STATUS: DEFINED — This can be used now even before inference works.
+# It's a plain data class with no dependencies.
+#
+# Usage:
+#   from models.chat_message import ChatMessage, Role
+#   msg = ChatMessage(role=Role.USER, content="Hello!")
+
+from enum import Enum
+from datetime import datetime, timezone
+
+
+class Role(str, Enum):
+    """
+    The role of the entity producing a message.
+    Mirrors the convention used in most chat template formats.
+    """
+    SYSTEM = "system"
+    USER = "user"
+    ASSISTANT = "assistant"
+
+    # Future: tool call results
+    # TOOL = "tool"
+
+
+class ChatMessage:
+    """
+    Represents a single message in a conversation.
+
+    Attributes:
+        role      (Role)     — Who produced this message
+        content   (str)      — The text content
+        timestamp (datetime) — When this message was created (UTC)
+        id        (str|None) — Optional unique ID for this message
+    """
+
+    def __init__(self, role: Role, content: str, timestamp: datetime | None = None, id: str | None = None):
+        self.role = role
+        self.content = content
+        self.timestamp = timestamp or datetime.now(timezone.utc)
+        self.id = id
+
+    def to_dict(self) -> dict:
+        return {
+            "role": self.role.value,
+            "content": self.content,
+            "timestamp": self.timestamp.isoformat(),
+            "id": self.id,
+        }
+
+    def __repr__(self) -> str:
+        preview = self.content[:60] + "..." if len(self.content) > 60 else self.content
+        return f"ChatMessage(role={self.role.value}, content='{preview}')"
+
+
+class Conversation:
+    """
+    A list of ChatMessages forming a conversation thread.
+
+    TODO (Phase 4.1): Add persistence, conversation ID, title generation.
+    """
+
+    def __init__(self, id: str | None = None):
+        self.id = id
+        self.messages: list[ChatMessage] = []
+
+    def add_message(self, message: ChatMessage) -> None:
+        self.messages.append(message)
+
+    def get_messages(self) -> list[ChatMessage]:
+        return self.messages
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "messages": [m.to_dict() for m in self.messages],
+        }

backend/models/model_config.py

@@ -0,0 +1,145 @@
+"""Data models for model configuration and inference I/O."""
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+CHAT_TEMPLATES = {
+    "auto",
+    "chatml",
+    "llama3",
+    "llama2",
+    "mistral",
+    "alpaca",
+    "raw",
+}
+
+BACKEND_TYPES = {
+    "llama-cpp",
+    "ollama",
+    "vllm",
+    "openai",
+    "anthropic",
+    "gemini",
+    "openai_compatible",
+}
+
+
+@dataclass
+class GenerationParams:
+    """Parameters controlling text generation."""
+    temperature: float = 0.7
+    top_p: float = 0.9
+    top_k: int = 40
+    repeat_penalty: float = 1.1
+    max_new_tokens: int = 512
+    min_new_tokens: int = 1
+    seed: int = -1
+
+    def to_dict(self) -> dict:
+        return {
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "top_k": self.top_k,
+            "repeat_penalty": self.repeat_penalty,
+            "max_new_tokens": self.max_new_tokens,
+            "min_new_tokens": self.min_new_tokens,
+            "seed": self.seed,
+        }
+
+
+@dataclass
+class ModelConfig:
+    """Complete configuration for one model — all from config, no hardcoded defaults."""
+    model_file: str = ""
+    backend_type: str = "llama-cpp"
+    chat_template: str = "auto"
+    context_length: int = 4096
+    system_prompt: str = "You are a helpful assistant."
+    n_gpu_layers: int = 0
+    generation: GenerationParams = field(default_factory=GenerationParams)
+    profile_name: Optional[str] = None
+    models_dir: str = "./models"
+    # API backend overrides
+    openai_model: str = ""
+    anthropic_model: str = ""
+    gemini_model: str = ""
+    ollama_model: str = ""
+    vllm_model: str = ""
+    compatible_model: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "profile_name": self.profile_name,
+            "backend_type": self.backend_type,
+            "model_file": self.model_file,
+            "chat_template": self.chat_template,
+            "context_length": self.context_length,
+            "system_prompt": self.system_prompt,
+            "n_gpu_layers": self.n_gpu_layers,
+            "models_dir": self.models_dir,
+            "generation": self.generation.to_dict(),
+            "openai_model": self.openai_model,
+            "anthropic_model": self.anthropic_model,
+            "gemini_model": self.gemini_model,
+            "ollama_model": self.ollama_model,
+            "vllm_model": self.vllm_model,
+            "compatible_model": self.compatible_model,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ModelConfig":
+        gen_data = data.get("generation", {})
+        generation = GenerationParams(
+            temperature=gen_data.get("temperature", 0.7),
+            top_p=gen_data.get("top_p", 0.9),
+            top_k=gen_data.get("top_k", 40),
+            repeat_penalty=gen_data.get("repeat_penalty", 1.1),
+            max_new_tokens=gen_data.get("max_new_tokens", 512),
+            seed=gen_data.get("seed", -1),
+        )
+        return cls(
+            model_file=data.get("model_file", ""),
+            backend_type=data.get("backend_type", "llama-cpp"),
+            chat_template=data.get("chat_template", "auto"),
+            context_length=data.get("context_length", 4096),
+            system_prompt=data.get("system_prompt", "You are a helpful assistant."),
+            n_gpu_layers=data.get("n_gpu_layers", 0),
+            generation=generation,
+            profile_name=data.get("profile_name"),
+            models_dir=data.get("models_dir", "./models"),
+            openai_model=data.get("openai_model", ""),
+            anthropic_model=data.get("anthropic_model", ""),
+            gemini_model=data.get("gemini_model", ""),
+            ollama_model=data.get("ollama_model", ""),
+            vllm_model=data.get("vllm_model", ""),
+            compatible_model=data.get("compatible_model", ""),
+        )
+
+
+@dataclass
+class InferenceRequest:
+    """Internal representation of an inference request."""
+    prompt: str
+    system_prompt: str = ""
+    max_tokens: int = 512
+    temperature: float = 0.7
+    top_p: float = 0.9
+    top_k: int = 40
+    repeat_penalty: float = 1.1
+    seed: int = -1
+    stop_tokens: list[str] = field(default_factory=list)
+
+
+@dataclass
+class InferenceResponse:
+    """Internal representation of an inference response."""
+    text: str
+    tokens_prompt: int = 0
+    tokens_generated: int = 0
+    finish_reason: str = "stop"
+    model_name: str = ""
+
+    @property
+    def tokens_used(self) -> int:
+        return self.tokens_prompt + self.tokens_generated

backend/routes/chat.py

@@ -0,0 +1,92 @@
+"""POST /chat and POST /chat/stream — Chat endpoints with streaming."""
+
+import sys
+from pathlib import Path
+
+_project_root = str(Path(__file__).resolve().parent.parent.parent)
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+import json
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+from controllers.chat_controller import handle_chat_request
+
+router = APIRouter()
+
+
+class ChatRequest(BaseModel):
+    message: str
+    conversation_id: str | None = None
+
+
+class ChatResponse(BaseModel):
+    response: str
+    model: str
+    tokens_used: int
+    finish_reason: str = "stop"
+
+
+@router.post("/", response_model=ChatResponse)
+async def chat(request: ChatRequest):
+    """Send a message and receive a response from any backend."""
+    if not request.message.strip():
+        raise HTTPException(status_code=400, detail="Message cannot be empty.")
+    return await handle_chat_request(request)
+
+
+@router.post("/stream")
+async def chat_stream(request: ChatRequest):
+    """Stream a response token-by-token using SSE."""
+    if not request.message.strip():
+        raise HTTPException(status_code=400, detail="Message cannot be empty.")
+
+    from models.chat_message import ChatMessage, Role
+    from services.inference_service import generate_stream_response
+    from services.conversation_service import (
+        get_conversation,
+        add_message_to_conversation,
+    )
+
+    conversation_history = None
+    if request.conversation_id:
+        conv = get_conversation(request.conversation_id)
+        if conv:
+            conversation_history = [
+                ChatMessage(role=Role(m["role"]), content=m["content"])
+                for m in conv["messages"]
+            ]
+
+    user_message = ChatMessage(role=Role.USER, content=request.message)
+
+    def event_generator():
+        full_response = []
+        try:
+            for token in generate_stream_response(
+                message=user_message,
+                conversation_history=conversation_history,
+            ):
+                full_response.append(token)
+                yield f"data: {json.dumps({'token': token, 'done': False})}\n\n"
+        except Exception as e:
+            yield f"data: {json.dumps({'error': str(e), 'done': True})}\n\n"
+            return
+
+        # Save to conversation
+        if request.conversation_id:
+            add_message_to_conversation(request.conversation_id, "user", request.message)
+            add_message_to_conversation(request.conversation_id, "assistant", "".join(full_response))
+
+        yield f"data: {json.dumps({'token': '', 'done': True})}\n\n"
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no",
+        },
+    )

backend/routes/config.py

@@ -0,0 +1,75 @@
+"""GET /config and PUT /config — Read/update active model configuration."""
+
+import sys
+from pathlib import Path
+
+_project_root = str(Path(__file__).resolve().parent.parent.parent)
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from models.model_config import ModelConfig, GenerationParams
+from services.model_service import get_active_config, set_active_config, scan_model_directory
+
+router = APIRouter()
+
+
+class ConfigUpdate(BaseModel):
+    backend_type: str | None = None
+    model_file: str | None = None
+    chat_template: str | None = None
+    context_length: int | None = None
+    system_prompt: str | None = None
+    n_gpu_layers: int | None = None
+    generation: dict | None = None
+    ollama_model: str | None = None
+    vllm_model: str | None = None
+
+
+@router.get("/")
+async def get_config():
+    config = get_active_config()
+    if config is None:
+        raise HTTPException(status_code=404, detail="No active model configuration.")
+    return config.to_dict()
+
+
+@router.put("/")
+async def update_config(update: ConfigUpdate):
+    config = get_active_config()
+    if config is None:
+        # Create new config from defaults
+        config = ModelConfig(model_file="")
+
+    if update.backend_type is not None:
+        config.backend_type = update.backend_type
+    if update.model_file is not None:
+        models = scan_model_directory()
+        filenames = [m["filename"] for m in models]
+        if update.model_file not in filenames:
+            raise HTTPException(status_code=404, detail=f"Model not found: {update.model_file}")
+        config.model_file = update.model_file
+    if update.chat_template is not None:
+        config.chat_template = update.chat_template
+    if update.context_length is not None:
+        config.context_length = update.context_length
+    if update.system_prompt is not None:
+        config.system_prompt = update.system_prompt
+    if update.n_gpu_layers is not None:
+        config.n_gpu_layers = update.n_gpu_layers
+    if update.generation is not None:
+        for key, val in update.generation.items():
+            if hasattr(config.generation, key):
+                setattr(config.generation, key, val)
+
+    set_active_config(config)
+    return {"message": "Config updated", "config": config.to_dict()}
+
+
+@router.get("/backends")
+async def list_backends():
+    """List all available BYOK backends and their status."""
+    from model_runtime.backends import list_backends
+    return {"backends": list_backends()}

backend/routes/conversations.py

@@ -0,0 +1,76 @@
+"""Conversation history routes with full CRUD."""
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from services.conversation_service import (
+    create_conversation,
+    get_conversation,
+    list_conversations,
+    delete_conversation,
+    add_message_to_conversation,
+    update_conversation_title,
+)
+
+router = APIRouter()
+
+
+class ConversationCreate(BaseModel):
+    title: str | None = None
+
+
+class ConversationRename(BaseModel):
+    title: str
+
+
+class MessageAdd(BaseModel):
+    role: str
+    content: str
+
+
+@router.get("/")
+async def get_all_conversations():
+    """List all conversations, sorted by most recent."""
+    return {"conversations": list_conversations()}
+
+
+@router.post("/")
+async def create_new_conversation(body: ConversationCreate):
+    """Create a new conversation."""
+    conv = create_conversation(title=body.title)
+    return conv
+
+
+@router.get("/{conversation_id}")
+async def get_conversation_detail(conversation_id: str):
+    """Get a conversation with all messages."""
+    conv = get_conversation(conversation_id)
+    if conv is None:
+        raise HTTPException(status_code=404, detail="Conversation not found.")
+    return conv
+
+
+@router.patch("/{conversation_id}")
+async def rename_conversation(conversation_id: str, body: ConversationRename):
+    """Rename a conversation."""
+    result = update_conversation_title(conversation_id, body.title)
+    if result is None:
+        raise HTTPException(status_code=404, detail="Conversation not found.")
+    return result
+
+
+@router.delete("/{conversation_id}")
+async def delete_conversation_route(conversation_id: str):
+    """Delete a conversation."""
+    ok = delete_conversation(conversation_id)
+    if not ok:
+        raise HTTPException(status_code=404, detail="Conversation not found.")
+    return {"message": "Deleted"}
+
+
+@router.post("/{conversation_id}/messages")
+async def add_message(conversation_id: str, body: MessageAdd):
+    """Add a message to a conversation."""
+    msg = add_message_to_conversation(conversation_id, body.role, body.content)
+    if msg is None:
+        raise HTTPException(status_code=404, detail="Conversation not found.")
+    return msg

backend/routes/metrics.py

@@ -0,0 +1,11 @@
+"""GET /metrics -- Performance monitoring endpoint."""
+
+from fastapi import APIRouter
+from services.metrics_service import get_metrics
+
+router = APIRouter()
+
+
+@router.get("/")
+async def metrics():
+    return get_metrics()

backend/routes/models.py

@@ -0,0 +1,29 @@
+"""GET /models — List available GGUF model files with auto-detected metadata."""
+
+import sys
+from pathlib import Path
+
+_project_root = str(Path(__file__).resolve().parent.parent.parent)
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from fastapi import APIRouter
+from services.model_service import scan_model_directory
+
+router = APIRouter()
+
+
+@router.get("/")
+async def list_models():
+    models = scan_model_directory()
+    return {"models": models}
+
+
+@router.get("/detect")
+async def detect_models():
+    """Auto-detect metadata for all GGUF files in models/ directory."""
+    from model_runtime.detector import detect_models_in_directory
+    from services.model_service import _get_models_dir
+    models_dir = _get_models_dir()
+    results = detect_models_in_directory(models_dir)
+    return {"models": results}

backend/routes/status.py

@@ -0,0 +1,24 @@
+"""GET /status — Health check and model state."""
+
+import sys
+from pathlib import Path
+
+# Ensure project root on sys.path
+_project_root = str(Path(__file__).resolve().parent.parent.parent)
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from fastapi import APIRouter
+from services.model_service import is_model_loaded, get_current_model_name
+
+router = APIRouter()
+
+
+@router.get("/")
+async def get_status():
+    return {
+        "status": "ok",
+        "model_loaded": is_model_loaded(),
+        "current_model": get_current_model_name(),
+        "version": "0.1.0",
+    }

backend/server.py

@@ -0,0 +1,41 @@
+"""
+Legacy server.py — kept for backward compat.
+The real entry point is local_ai_runtime/cli.py -> local_ai_runtime/server.py.
+This file exists so `python backend/server.py` still works.
+"""
+
+import sys
+import json
+import logging
+from pathlib import Path
+
+# Add project root to sys.path
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+import uvicorn
+
+CONFIG_DIR = _PROJECT_ROOT / "config"
+SERVER_CONFIG_PATH = CONFIG_DIR / "server.config.json"
+
+
+def load_server_config() -> dict:
+    if SERVER_CONFIG_PATH.exists():
+        with open(SERVER_CONFIG_PATH) as f:
+            return json.load(f)
+    raise FileNotFoundError(f"Config not found: {SERVER_CONFIG_PATH}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    )
+    config = load_server_config()
+    uvicorn.run(
+        "backend.server:app",
+        host=config.get("host", "0.0.0.0"),
+        port=config.get("port", 8000),
+        log_level=config.get("log_level", "info"),
+    )