lemonade-sdk 8.1.11__py3-none-any.whl → 8.2.2__py3-none-any.whl

lemonade/tools/server/serve.py

@@ -7,12 +7,12 @@ import logging
 import platform
 import tempfile
 import traceback
-from typing import Optional, Union
+from typing import Optional, Union, List
 import json
 from pathlib import Path
 import os
-
-from fastapi import FastAPI, HTTPException, status, Request, WebSocket
+import shutil
+from fastapi import FastAPI, HTTPException, status, Request, WebSocket, Form, UploadFile
 from fastapi.responses import StreamingResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
@@ -83,10 +83,31 @@ if platform.system() in ["Windows", "Darwin"]:
     from lemonade.tools.server.tray import LemonadeTray, OutputDuplicator
 
 
-class WebsocketTextFilter(logging.Filter):
+class ServerLogFilter(logging.Filter):
+    def __init__(self, server):
+        super().__init__()
+        self.server = server
+        self.noisy_paths = {
+            "/api/v1/health",
+            "/api/v0/health",
+            "/api/v1/models",
+            "/api/v0/models",
+        }
+
     def filter(self, record: logging.LogRecord) -> bool:
-        # Only allow logs that don't include "> TEXT"
-        return "> TEXT" not in record.getMessage()
+        msg = record.getMessage()
+
+        # Filter out websocket logs
+        if "> TEXT" in msg:
+            return False
+
+        # Filter out noisy HTTP routes if debug logs are OFF
+        if not self.server.debug_logging_enabled:
+            if any(path in msg for path in self.noisy_paths):
+                return False
+
+        # Otherwise, allow the log
+        return True
 
 
 async def log_streamer(websocket: WebSocket, path: str, interval: float = 1.0):
@@ -94,7 +115,7 @@ async def log_streamer(websocket: WebSocket, path: str, interval: float = 1.0):
     await websocket.accept()
     try:
         with open(path, "r", encoding="utf-8") as f:
-            f.seek(0, os.SEEK_END)  # start at end
+            f.seek(0)  # start at the beginning of the file
             while True:
                 # Try reading a line
                 line = f.readline()
@@ -325,6 +346,7 @@ class Server:
             self.app.post(f"{prefix}/responses")(self.responses)
             self.app.post(f"{prefix}/log-level")(self.set_log_level)
             self.app.websocket(f"{prefix}/logs/ws")(self.logs_ws)
+            self.app.post(f"{prefix}/add-local-model")(self.add_local_model)
 
             # OpenAI-compatible routes
             self.app.post(f"{prefix}/chat/completions")(self.chat_completions)
@@ -336,6 +358,186 @@ class Server:
             self.app.post(f"{prefix}/reranking")(self.reranking)
             self.app.post(f"{prefix}/rerank")(self.reranking)
 
+            # Migration routes
+            self.app.get(f"{prefix}/migration/incompatible-models")(
+                self.get_incompatible_models
+            )
+            self.app.post(f"{prefix}/migration/cleanup")(
+                self.cleanup_incompatible_models
+            )
+
+    async def add_local_model(
+        self,
+        model_name: str = Form(...),
+        checkpoint: str = Form(""),
+        recipe: str = Form(...),
+        reasoning: bool = Form(False),
+        vision: bool = Form(False),
+        mmproj: str = Form(None),
+        model_files: List[UploadFile] = None,
+    ):
+        from huggingface_hub.constants import HF_HUB_CACHE
+        from lemonade.tools.llamacpp.utils import parse_checkpoint
+
+        # Upload and register a local model from files.
+        try:
+            if not model_files:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail="No model files provided for upload",
+                )
+
+            if not model_name.startswith("user."):
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail="Model name must start with 'user.'",
+                )
+
+            valid_recipes = ["llamacpp", "oga-npu", "oga-hybrid", "oga-cpu"]
+            if recipe not in valid_recipes:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=f"Invalid recipe. Must be one of: {', '.join(valid_recipes)}",
+                )
+
+            if recipe == "llamacpp" and not any(
+                f.filename.lower().endswith(".gguf") for f in model_files
+            ):
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail="At least one .gguf file is required for llamacpp",
+                )
+
+            # Check if model name already exists
+            if model_name in ModelManager().supported_models:
+                raise HTTPException(
+                    status_code=status.HTTP_409_CONFLICT,
+                    detail=(
+                        f"Model name '{model_name}' already exists. "
+                        "Please use a different name."
+                    ),
+                )
+
+            model_name_clean = model_name.replace("user.", "")
+
+            # Files are saved to models--{model_name_clean}
+            # Note: This is based on the user's custom model name, NOT the checkpoint field
+            repo_cache_name = model_name_clean.replace("/", "--")
+            snapshot_path = os.path.join(HF_HUB_CACHE, f"models--{repo_cache_name}")
+            os.makedirs(snapshot_path, exist_ok=True)
+
+            # Extract variant from checkpoint field if provided
+            # checkpoint field format: "folder:variant" or just "folder"
+            variant = None
+            if checkpoint and ":" in checkpoint:
+                _, variant = parse_checkpoint(checkpoint)
+                # variant now contains just the variant[can be with or without the
+                # .gguf extension] filename (e.g., "LFM2-VL-1.6B-F16 or LFM2-VL-1.6B-F16.gguf")
+
+            # Save uploaded files, preserving folder structure
+            for file in model_files:
+                relative_path = file.filename
+                path_parts = relative_path.split("/")
+
+                if len(path_parts) > 1:
+                    internal_path = "/".join(path_parts[1:])
+                    file_path = os.path.join(snapshot_path, internal_path)
+                else:
+                    file_path = os.path.join(snapshot_path, path_parts[0])
+
+                os.makedirs(os.path.dirname(file_path), exist_ok=True)
+                with open(file_path, "wb") as f:
+                    content = await file.read()
+                    f.write(content)
+
+            # Resolve actual file paths after upload (for faster loading later)
+            resolved_checkpoint = None
+            resolved_mmproj = None
+
+            # For OGA models, find genai_config.json
+            if recipe.startswith("oga-"):
+                for root, _, files in os.walk(snapshot_path):
+                    if "genai_config.json" in files:
+                        resolved_checkpoint = root
+                        break
+                if not resolved_checkpoint:
+                    resolved_checkpoint = snapshot_path
+
+            # For llamacpp models, find the GGUF file
+            elif recipe == "llamacpp":
+                gguf_file_found = None
+
+                # If variant is specified, look for that specific file
+                if variant:
+                    search_term = (
+                        variant if variant.endswith(".gguf") else f"{variant}.gguf"
+                    )
+                    for root, _, files in os.walk(snapshot_path):
+                        if search_term in files:
+                            gguf_file_found = os.path.join(root, search_term)
+                            break
+
+                # If no variant or variant not found, search for any .gguf file (excluding mmproj)
+                if not gguf_file_found:
+                    for root, _, files in os.walk(snapshot_path):
+                        gguf_files = [
+                            f
+                            for f in files
+                            if f.endswith(".gguf") and "mmproj" not in f.lower()
+                        ]
+                        if gguf_files:
+                            gguf_file_found = os.path.join(root, gguf_files[0])
+                            break
+
+                resolved_checkpoint = (
+                    gguf_file_found if gguf_file_found else snapshot_path
+                )
+
+            # Search for mmproj file if provided
+            if mmproj:
+                for root, _, files in os.walk(snapshot_path):
+                    if mmproj in files:
+                        resolved_mmproj = os.path.join(root, mmproj)
+                        break
+
+            # Build checkpoint for registration
+            # For llamacpp with resolved path, store the full path relative to HF_HUB_CACHE
+            if resolved_checkpoint:
+                # Store as relative path from HF_HUB_CACHE for portability
+                checkpoint_to_register = os.path.relpath(
+                    resolved_checkpoint, HF_HUB_CACHE
+                )
+            elif variant:
+                checkpoint_to_register = f"models--{repo_cache_name}:{variant}"
+            else:
+                checkpoint_to_register = f"models--{repo_cache_name}"
+
+            # Register the model
+            ModelManager().register_local_model(
+                model_name=model_name,
+                checkpoint=checkpoint_to_register,
+                recipe=recipe,
+                reasoning=reasoning,
+                vision=vision,
+                mmproj=resolved_mmproj if resolved_mmproj else mmproj,
+                snapshot_path=snapshot_path,
+            )
+
+            # Refresh local models
+            self.local_models = ModelManager().downloaded_models_enabled
+
+            return {
+                "status": "success",
+                "message": f"Model {model_name} uploaded and registered successfully",
+            }
+        except Exception as e:
+            if os.path.exists(checkpoint_to_register):
+                shutil.rmtree(checkpoint_to_register)
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Failed to upload model: {str(e)}",
+            )
+
     async def set_log_level(self, config: LogLevelConfig):
         """
         Set the logging level of the server.
@@ -454,13 +656,13 @@ class Server:
             )
             file_handler.setLevel(logging_level)
             file_handler.setFormatter(uvicorn_formatter)
-            file_handler.addFilter(WebsocketTextFilter())
+            file_handler.addFilter(ServerLogFilter(self))
 
             # Set up console handler
             console_handler = logging.StreamHandler()
             console_handler.setLevel(logging_level)
             console_handler.setFormatter(uvicorn_formatter)
-            console_handler.addFilter(WebsocketTextFilter())
+            console_handler.addFilter(ServerLogFilter(self))
 
             # Configure root logger with both handlers
             logging.basicConfig(
@@ -746,6 +948,23 @@ class Server:
         await self.load_llm(lc)
 
         if self.llm_loaded.recipe == "llamacpp" or self.llm_loaded.recipe == "flm":
+            if (
+                hasattr(chat_completion_request, "enable_thinking")
+                and chat_completion_request.enable_thinking is False
+                and "qwen3" in self.llm_loaded.model_name.lower()
+            ):
+
+                # Modify the last user message to include /no_think
+                if chat_completion_request.messages:
+                    for i in range(len(chat_completion_request.messages) - 1, -1, -1):
+                        if chat_completion_request.messages[i].get("role") == "user":
+                            original_content = chat_completion_request.messages[i][
+                                "content"
+                            ]
+                            chat_completion_request.messages[i][
+                                "content"
+                            ] = f"/no_think\n{original_content}"
+                            break
             return self.wrapped_server.chat_completion(chat_completion_request)
 
         # Convert chat messages to text using the model's chat template
@@ -807,6 +1026,11 @@ class Server:
 
                 # Keep track of the full response for tool call extraction
                 full_response = ""
+
+                # Track whether we're still in the thinking phase (before </think> tag)
+                in_thinking_phase = self.llm_loaded.reasoning
+                reasoning_buffer = ""  # Accumulate reasoning tokens to detect </think>
+
                 try:
                     async for token in self._generate_tokens(**generation_args):
                         # Handle client disconnect: stop generation and exit
@@ -845,7 +1069,53 @@ class Server:
                                     )
                                 )
 
-                        # Create a ChatCompletionChunk
+                        # Create a ChatCompletionChunk with reasoning_content support
+                        # If we're in reasoning mode and haven't seen </think> yet,
+                        # send tokens as reasoning_content instead of content
+                        delta_content = None
+                        delta_reasoning = None
+
+                        if reasoning_first_token:
+                            # First token - include opening tag in reasoning
+                            delta_reasoning = "<think>" + token
+                            reasoning_first_token = False
+                            reasoning_buffer = token
+                        elif in_thinking_phase:
+                            # Still in thinking phase - accumulate and check for </think>
+                            reasoning_buffer += token
+
+                            # Check if we've seen the closing tag
+                            if "</think>" in reasoning_buffer:
+                                # Split at the closing tag
+                                before_close, after_close = reasoning_buffer.split(
+                                    "</think>", 1
+                                )
+
+                                # Send everything before + closing tag as reasoning
+                                if before_close or not reasoning_buffer.startswith(
+                                    "</think>"
+                                ):
+                                    delta_reasoning = before_close + "</think>"
+                                else:
+                                    delta_reasoning = "</think>"
+
+                                # Everything after goes to content (will be sent in next iteration)
+                                # For now, mark that we've exited thinking phase
+                                in_thinking_phase = False
+
+                                # If there's content after </think>, we need to send it too
+                                # But we send it in the current chunk as regular content
+                                if after_close:
+                                    # We have both reasoning and content in this token
+                                    # Send reasoning first, content will accumulate
+                                    delta_content = after_close
+                            else:
+                                # Still accumulating thinking, send as reasoning_content
+                                delta_reasoning = token
+                        else:
+                            # Normal content (after thinking phase ended)
+                            delta_content = token
+
                         chunk = ChatCompletionChunk.model_construct(
                             id="0",
                             object="chat.completion.chunk",
@@ -855,11 +1125,8 @@ class Server:
                                 Choice.model_construct(
                                     index=0,
                                     delta=ChoiceDelta(
-                                        content=(
-                                            "<think>" + token
-                                            if reasoning_first_token
-                                            else token
-                                        ),
+                                        content=delta_content,
+                                        reasoning_content=delta_reasoning,
                                         function_call=None,
                                         role="assistant",
                                         tool_calls=openai_tool_calls,
@@ -872,7 +1139,6 @@ class Server:
                         )
 
                         # Format as SSE
-                        reasoning_first_token = False
                         yield f"data: {chunk.model_dump_json()}\n\n".encode("utf-8")
 
                     # Send the [DONE] marker only if still connected
@@ -1125,9 +1391,10 @@ class Server:
                                 "<think>" + token if reasoning_first_token else token
                             ),
                             item_id="0 ",
+                            logprobs=[],
                             output_index=0,
-                            type="response.output_text.delta",
                             sequence_number=0,
+                            type="response.output_text.delta",
                         )
                         full_response += token
 
@@ -1570,9 +1837,10 @@ class Server:
         Load a registered LLM into system memory. Install the model first, if needed.
             config: the information required to load the model
         """
+        from huggingface_hub.constants import HF_HUB_CACHE
+
         try:
             await self._load_lock.acquire()
-
             # Acquire all generate locks
             for _ in range(self.max_concurrent_generations):
                 await self._generate_semaphore.acquire()
@@ -1597,6 +1865,38 @@ class Server:
                 # Get additional properties from the model registry
                 config_to_use = LoadConfig(**supported_models[config.model_name])
 
+            # For locally uploaded models, convert the relative checkpoint path to absolute path
+            model_source = supported_models.get(config.model_name, {}).get(
+                "source", None
+            )
+            if (
+                model_source == "local_upload"
+                and config_to_use.checkpoint
+                and not config_to_use.recipe.startswith("hf-")
+            ):
+                # Check if checkpoint is a relative path (stored during upload)
+                if not os.path.isabs(config_to_use.checkpoint):
+                    # Convert relative path to absolute by joining with HF_HUB_CACHE
+                    absolute_checkpoint = os.path.join(
+                        HF_HUB_CACHE, config_to_use.checkpoint
+                    )
+                    if os.path.exists(absolute_checkpoint):
+                        config_to_use.checkpoint = absolute_checkpoint
+                    else:
+                        logging.warning(
+                            f"Checkpoint path does not exist: {absolute_checkpoint}"
+                        )
+
+                # Also resolve mmproj path if present
+                if config_to_use.mmproj and not os.path.isabs(config_to_use.mmproj):
+                    absolute_mmproj = os.path.join(HF_HUB_CACHE, config_to_use.mmproj)
+                    if os.path.exists(absolute_mmproj):
+                        config_to_use.mmproj = absolute_mmproj
+                    else:
+                        logging.warning(
+                            f"MMProj path does not exist: {absolute_mmproj}"
+                        )
+
             # Caching mechanism: if the checkpoint is already loaded there is nothing else to do
             if (
                 self.llm_loaded
@@ -1782,6 +2082,42 @@ class Server:
             return
         await log_streamer(websocket, self.log_file)
 
+    async def get_incompatible_models(self):
+        """
+        Get information about incompatible RyzenAI models in the cache.
+        """
+        try:
+            return ModelManager().get_incompatible_ryzenai_models()
+        except Exception as e:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Failed to scan for incompatible models: {str(e)}",
+            )
+
+    async def cleanup_incompatible_models(self, request: Request):
+        """
+        Delete selected incompatible RyzenAI models from the cache.
+        """
+        try:
+            body = await request.json()
+            model_paths = body.get("model_paths", [])
+
+            if not model_paths:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail="No model_paths provided",
+                )
+
+            result = ModelManager().cleanup_incompatible_models(model_paths)
+            return result
+        except HTTPException:
+            raise
+        except Exception as e:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Failed to cleanup models: {str(e)}",
+            )
+
 
 # This file was originally licensed under Apache 2.0. It has been modified.
 # Modifications Copyright (c) 2025 AMD

lemonade/tools/server/static/js/chat.js

@@ -4,9 +4,6 @@ let attachedFiles = [];
 let systemMessageElement = null;
 let abortController = null;
 
-// Default model configuration
-const DEFAULT_MODEL = 'Qwen2.5-0.5B-Instruct-CPU';
-
 const THINKING_ANIM_INTERVAL_MS = 550;
 // Toggle this to false if you prefer plain dots only.
 const THINKING_USE_LEMON = true;
@@ -165,17 +162,22 @@ async function handleModelSelectChange() {
             	loadingOption.hidden = true;
             	select.appendChild(loadingOption);
             }
+            // Gray out send button during loading
+            updateAttachmentButtonState();
         },
         onLoadingEnd: (modelId, success) => {
             // Reset the default option text
             const defaultOption = modelSelect.querySelector('option[value=""]');
             if (defaultOption) defaultOption.textContent = 'Click to select a model ▼';
+            // Update button state after loading completes
+            updateAttachmentButtonState();
 		},
         onSuccess: () => {
             updateAttachmentButtonState();
         },
         onError: () => {
             updateModelSelectValue();
+            updateAttachmentButtonState();
         }
     });
 }
@@ -192,7 +194,8 @@ function updateAttachmentButtonState() {
             toggleBtn.disabled = false;
             toggleBtn.textContent = 'Stop';
         } else {
-            toggleBtn.disabled = loading;
+            // Gray out send button if no model is loaded or if loading
+            toggleBtn.disabled = loading || !currentLoadedModel;
             toggleBtn.textContent = 'Send';
         }
     }
@@ -225,43 +228,6 @@ window.updateAttachmentButtonState = updateAttachmentButtonState;
 // Make displaySystemMessage accessible globally
 window.displaySystemMessage = displaySystemMessage;
 
-// Auto-load default model and send message
-async function autoLoadDefaultModelAndSend() {
-    // Check if default model is available and installed
-    if (!window.SERVER_MODELS || !window.SERVER_MODELS[DEFAULT_MODEL]) {
-        showErrorBanner('No models available. Please install a model first.');
-        return;
-    }
-
-    if (!window.installedModels || !window.installedModels.has(DEFAULT_MODEL)) {
-        showErrorBanner('Default model is not installed. Please install it from the Model Management tab.');
-        return;
-    }
-
-    // Store the message to send after loading
-    const messageToSend = chatInput.value.trim();
-    if (!messageToSend && attachedFiles.length === 0) return;
-
-    // Use the standardized load function
-    const success = await loadModelStandardized(DEFAULT_MODEL, {
-        // Custom UI updates for auto-loading
-        onLoadingStart: () => {
-            if (toggleBtn) {
-                toggleBtn.disabled = true;
-                toggleBtn.textContent = 'Send';
-            }
-        },
-        // Reset send button state
-        onLoadingEnd: () => { updateAttachmentButtonState(); },
-        // Send the message after successful load
-        onSuccess: () => { sendMessage(messageToSend); },
-        onError: (error) => {
-            console.error('Error auto-loading default model:', error);
-            showErrorBanner('Failed to load model: ' + error.message);
-        }
-    });
-}
-
 // Check if model supports vision and update attachment button
 function checkCurrentModel() {
     if (attachedFiles.length > 0 && currentLoadedModel && !isVisionModel(currentLoadedModel)) {
@@ -320,13 +286,11 @@ function handleChatInputKeydown(e) {
         clearAttachments();
     } else if (e.key === 'Enter' && !e.shiftKey) {
         e.preventDefault();
-        // Check if we have a loaded model
+        // Only send if we have a loaded model
         if (currentLoadedModel && modelSelect.value !== '' && !modelSelect.disabled) {
             sendMessage();
-        } else if (!currentLoadedModel) {
-            // Auto-load default model and send
-            autoLoadDefaultModelAndSend();
         }
+        // Otherwise do nothing - button is grayed out
     }
 }
 
@@ -860,39 +824,13 @@ async function sendMessage(existingTextIfAny) {
         systemMessageElement = null;
     }
 
-    // Check if a model is loaded, if not, automatically load the default model
+    // Check if a model is loaded
     if (!currentLoadedModel) {
-        const allModels = window.SERVER_MODELS || {};
-
-        if (allModels[DEFAULT_MODEL]) {
-            try {
-                // Show loading message
-                const loadingBubble = appendMessage('system', 'Loading default model, please wait...');
-
-                // Load the default model
-                await httpRequest(getServerBaseUrl() + '/api/v1/load', {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({ model_name: DEFAULT_MODEL })
-                });
-
-                // Update model status
-                await updateModelStatusIndicator();
-
-                // Remove loading message
-                loadingBubble.parentElement.remove();
-
-                // Show success message briefly
-                const successBubble = appendMessage('system', `Loaded ${DEFAULT_MODEL} successfully!`);
-                setTimeout(() => { successBubble.parentElement.remove(); }, 2000);
-            } catch (error) {
-                alert('Please load a model first before sending messages.');
-                return;
-            }
-        } else {
-            alert('Please load a model first before sending messages.');
-            return;
-        }
+        alert('Please load a model first before sending messages.');
+        abortController = null;
+        isStreaming = false;
+        updateAttachmentButtonState();
+        return;
     }
 
     // Check if trying to send images to non-vision model