lemonade-sdk 7.0.0__tar.gz → 7.0.1__tar.gz

{lemonade_sdk-7.0.0 → lemonade_sdk-7.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lemonade-sdk
-Version: 7.0.0
+Version: 7.0.1
 Summary: Lemonade SDK: Your LLM Aide for Validation and Deployment
 Author-email: lemonade@amd.com
 Requires-Python: >=3.10, <3.12
@@ -25,8 +25,8 @@ Requires-Dist: matplotlib
 Requires-Dist: tabulate
 Requires-Dist: huggingface-hub==0.30.2
 Provides-Extra: llm
-Requires-Dist: torch>=2.0.0; extra == "llm"
-Requires-Dist: transformers; extra == "llm"
+Requires-Dist: torch>=2.6.0; extra == "llm"
+Requires-Dist: transformers<=4.51.3; extra == "llm"
 Requires-Dist: accelerate; extra == "llm"
 Requires-Dist: py-cpuinfo; extra == "llm"
 Requires-Dist: sentencepiece; extra == "llm"
@@ -34,23 +34,20 @@ Requires-Dist: datasets; extra == "llm"
 Requires-Dist: human-eval-windows==1.0.4; extra == "llm"
 Requires-Dist: fastapi; extra == "llm"
 Requires-Dist: uvicorn[standard]; extra == "llm"
-Requires-Dist: openai>=1.66.0; extra == "llm"
+Requires-Dist: openai>=1.81.0; extra == "llm"
 Requires-Dist: lm-eval[api]; extra == "llm"
 Provides-Extra: llm-oga-cpu
 Requires-Dist: onnxruntime-genai==0.6.0; extra == "llm-oga-cpu"
 Requires-Dist: onnxruntime<1.22.0,>=1.10.1; extra == "llm-oga-cpu"
-Requires-Dist: torch<2.4,>=2.0.0; extra == "llm-oga-cpu"
 Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-cpu"
 Provides-Extra: llm-oga-igpu
 Requires-Dist: onnxruntime-genai-directml==0.6.0; extra == "llm-oga-igpu"
 Requires-Dist: onnxruntime-directml<1.22.0,>=1.19.0; extra == "llm-oga-igpu"
-Requires-Dist: torch<2.4,>=2.0.0; extra == "llm-oga-igpu"
 Requires-Dist: transformers<4.45.0; extra == "llm-oga-igpu"
 Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-igpu"
 Provides-Extra: llm-oga-cuda
 Requires-Dist: onnxruntime-genai-cuda==0.6.0; extra == "llm-oga-cuda"
 Requires-Dist: onnxruntime-gpu<1.22.0,>=1.19.1; extra == "llm-oga-cuda"
-Requires-Dist: torch<2.4,>=2.0.0; extra == "llm-oga-cuda"
 Requires-Dist: transformers<4.45.0; extra == "llm-oga-cuda"
 Requires-Dist: lemonade-sdk[llm]; extra == "llm-oga-cuda"
 Provides-Extra: llm-oga-npu

{lemonade_sdk-7.0.0 → lemonade_sdk-7.0.1}/setup.py

@@ -3,7 +3,6 @@ from setuptools import setup
 with open("src/lemonade/version.py", encoding="utf-8") as fp:
     version = fp.read().split('"')[1]
 
-
 setup(
     name="lemonade-sdk",
     version=version,
@@ -46,8 +45,8 @@ setup(
     ],
     extras_require={
         "llm": [
-            "torch>=2.0.0",
-            "transformers",
+            "torch>=2.6.0",
+            "transformers<=4.51.3",
             "accelerate",
             "py-cpuinfo",
             "sentencepiece",
@@ -57,26 +56,23 @@ setup(
             "human-eval-windows==1.0.4",
             "fastapi",
             "uvicorn[standard]",
-            "openai>=1.66.0",
+            "openai>=1.81.0",
             "lm-eval[api]",
         ],
         "llm-oga-cpu": [
             "onnxruntime-genai==0.6.0",
             "onnxruntime >=1.10.1,<1.22.0",
-            "torch>=2.0.0,<2.4",
             "lemonade-sdk[llm]",
         ],
         "llm-oga-igpu": [
             "onnxruntime-genai-directml==0.6.0",
             "onnxruntime-directml>=1.19.0,<1.22.0",
-            "torch>=2.0.0,<2.4",
             "transformers<4.45.0",
             "lemonade-sdk[llm]",
         ],
         "llm-oga-cuda": [
             "onnxruntime-genai-cuda==0.6.0",
             "onnxruntime-gpu >=1.19.1,<1.22.0",
-            "torch>=2.0.0,<2.4",
             "transformers<4.45.0",
             "lemonade-sdk[llm]",
         ],
@@ -111,6 +107,7 @@ setup(
     include_package_data=True,
     package_data={
         "lemonade_server": ["server_models.json"],
+        "lemonade": ["tools/server/static/styles.css"],
     },
 )
 

lemonade_sdk-7.0.1/src/lemonade/tools/server/instructions.py

@@ -0,0 +1,294 @@
+from pathlib import Path
+import json
+from fastapi.responses import HTMLResponse
+from lemonade_server.model_manager import ModelManager
+
+
+def get_instructions_html(port=8000):
+    """
+    Show instructions on how to use the server.
+    """
+    # Load server models from JSON
+    server_models_path = (
+        Path(__file__).parent.parent.parent.parent
+        / "lemonade_server"
+        / "server_models.json"
+    )
+    with open(server_models_path, "r", encoding="utf-8") as f:
+        server_models = json.load(f)
+
+    # Use shared filter function from model_manager.py
+    filtered_models = ModelManager().filter_models_by_backend(server_models)
+
+    # Pass filtered server_models to JS
+    server_models_js = (
+        f"<script>window.SERVER_MODELS = {json.dumps(filtered_models)};</script>"
+    )
+
+    # New lemon-themed HTML structure
+    # pylint: disable=W1401
+    styled_html = f"""
+    <!DOCTYPE html>
+    <html lang=\"en\">
+    <head>
+        <meta charset=\"UTF-8\">
+        <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">
+        <title>Lemonade Server</title>
+        <link rel="icon" href="data:,">
+        <link rel=\"stylesheet\" href=\"/static/styles.css\">
+        <script>
+        window.SERVER_PORT = {port};
+        </script>
+        {server_models_js}
+    </head>
+    <body>
+        <nav class=\"navbar\">
+            <a href=\"https://github.com/lemonade-sdk/lemonade\">GitHub</a>
+            <a href=\"https://lemonade-server.ai/docs/\">Docs</a>
+            <a href=\"https://lemonade-server.ai/docs/server/server_models/\">Models</a>
+            <a href=\"https://lemonade-server.ai/docs/server/apps/\">Featured Apps</a>
+        </nav>
+        <main class=\"main\">
+            <div class=\"title\">🍋 Lemonade Server</div>
+            <div class=\"tab-container\"> 
+                <div class=\"tabs\"> 
+                    <button class=\"tab active\" id=\"tab-chat\" onclick=\"showTab('chat')\">LLM Chat</button> 
+                    <button class=\"tab\" id=\"tab-models\" onclick=\"showTab('models')\">Model Management</button> 
+                </div> 
+                <div class=\"tab-content active\" id=\"content-chat\"> 
+                    <div class=\"chat-container\"> 
+                        <div class=\"chat-history\" id=\"chat-history\"></div> 
+                        <div class=\"chat-input-row\"> 
+                            <select id=\"model-select\"></select> 
+                            <input type=\"text\" id=\"chat-input\" placeholder=\"Type your message...\" /> 
+                            <button id=\"send-btn\">Send</button> 
+                        </div> 
+                    </div> 
+                </div> 
+                <div class=\"tab-content\" id=\"content-models\"> 
+                    <div class=\"model-mgmt-container\">
+                        <div class=\"model-mgmt-pane\">
+                            <h3>Installed Models</h3>
+                            <table class=\"model-table\" id=\"installed-models-table\">
+                                <colgroup><col style=\"width:100%\"></colgroup>
+                                <tbody id=\"installed-models-tbody\"></tbody>
+                            </table>
+                        </div>
+                        <div class=\"model-mgmt-pane\">
+                            <h3>Suggested Models</h3>
+                            <table class=\"model-table\" id=\"suggested-models-table\">
+                                <tbody id=\"suggested-models-tbody\"></tbody>
+                            </table>
+                        </div>
+                    </div>
+                </div> 
+            </div> 
+        </main>
+        <footer class=\"site-footer\">
+            <div class=\"dad-joke\">When life gives you LLMs, make an LLM aide.</div>
+            <div class=\"copyright\">Copyright 2025 AMD</div>
+        </footer>
+        <script src=\"https://cdn.jsdelivr.net/npm/openai@4.21.0/dist/openai.min.js\"></script> 
+        <script> 
+        // Tab switching logic 
+        function showTab(tab) {{ 
+            document.getElementById('tab-chat').classList.remove('active'); 
+            document.getElementById('tab-models').classList.remove('active'); 
+            document.getElementById('content-chat').classList.remove('active'); 
+            document.getElementById('content-models').classList.remove('active'); 
+            if (tab === 'chat') {{ 
+                document.getElementById('tab-chat').classList.add('active'); 
+                document.getElementById('content-chat').classList.add('active'); 
+            }} else {{ 
+                document.getElementById('tab-models').classList.add('active'); 
+                document.getElementById('content-models').classList.add('active'); 
+            }} 
+        }}
+
+        // Helper to get server base URL
+        function getServerBaseUrl() {{
+            const port = window.SERVER_PORT || 8000;
+            return `http://localhost:{port}`;
+        }}
+
+        // Populate model dropdown from /api/v1/models endpoint
+        async function loadModels() {{
+            try {{
+                const resp = await fetch(getServerBaseUrl() + '/api/v1/models');
+                const data = await resp.json();
+                const select = document.getElementById('model-select');
+                select.innerHTML = '';
+                if (!data.data || !Array.isArray(data.data)) {{
+                    select.innerHTML = '<option>No models found (malformed response)</option>';
+                    return;
+                }}
+                if (data.data.length === 0) {{
+                    select.innerHTML = '<option>No models available</option>';
+                    return;
+                }}
+                let defaultIndex = 0;
+                data.data.forEach(function(model, index) {{
+                    const modelId = model.id || model.name || model;
+                    const opt = document.createElement('option');
+                    opt.value = modelId;
+                    opt.textContent = modelId;
+                    if (modelId === 'Llama-3.2-1B-Instruct-Hybrid') {{
+                        defaultIndex = index;
+                    }}
+                    select.appendChild(opt);
+                }});
+                select.selectedIndex = defaultIndex;
+            }} catch (e) {{
+                const select = document.getElementById('model-select');
+                select.innerHTML = `<option>Error loading models: ${{e.message}}</option>`;
+                console.error('Error loading models:', e);
+            }}
+        }}
+        loadModels();
+
+        // Model Management Tab Logic
+        async function refreshModelMgmtUI() {{
+            // Get installed models from /api/v1/models
+            let installed = [];
+            try {{
+                const resp = await fetch(getServerBaseUrl() + '/api/v1/models');
+                const data = await resp.json();
+                if (data.data && Array.isArray(data.data)) {{
+                    installed = data.data.map(m => m.id || m.name || m);
+                }}
+            }} catch (e) {{}}
+            // All models from server_models.json (window.SERVER_MODELS)
+            const allModels = window.SERVER_MODELS || {{}};
+            // Filter suggested models not installed
+            const suggested = Object.keys(allModels).filter(
+                k => allModels[k].suggested && !installed.includes(k)
+            );
+            // Render installed models as a table (two columns, second is invisible)
+            const installedTbody = document.getElementById('installed-models-tbody');
+            installedTbody.innerHTML = '';
+            installed.forEach(function(mid) {{
+                var tr = document.createElement('tr');
+                var tdName = document.createElement('td');
+                tdName.textContent = mid;
+                var tdEmpty = document.createElement('td');
+                tdEmpty.style.width = '0';
+                tdEmpty.style.padding = '0';
+                tdEmpty.style.border = 'none';
+                tr.appendChild(tdName);
+                tr.appendChild(tdEmpty);
+                installedTbody.appendChild(tr);
+            }});
+            // Render suggested models as a table
+            const suggestedTbody = document.getElementById('suggested-models-tbody');
+            suggestedTbody.innerHTML = '';
+            suggested.forEach(mid => {{
+                const tr = document.createElement('tr');
+                const tdName = document.createElement('td');
+                tdName.textContent = mid;
+                tdName.style.paddingRight = '1em';
+                tdName.style.verticalAlign = 'middle';
+                const tdBtn = document.createElement('td');
+                tdBtn.style.width = '1%';
+                tdBtn.style.verticalAlign = 'middle';
+                const btn = document.createElement('button');
+                btn.textContent = '+';
+                btn.title = 'Install model';
+                btn.onclick = async function() {{
+                    btn.disabled = true;
+                    btn.textContent = 'Installing...';
+                    btn.classList.add('installing-btn');
+                    try {{
+                        await fetch(getServerBaseUrl() + '/api/v1/pull', {{
+                            method: 'POST',
+                            headers: {{ 'Content-Type': 'application/json' }},
+                            body: JSON.stringify({{ model_name: mid }})
+                        }});
+                        await refreshModelMgmtUI();
+                        await loadModels(); // update chat dropdown too
+                    }} catch (e) {{
+                        btn.textContent = 'Error';
+                    }}
+                }};
+                tdBtn.appendChild(btn);
+                tr.appendChild(tdName);
+                tr.appendChild(tdBtn);
+                suggestedTbody.appendChild(tr);
+            }});
+        }}
+        // Initial load
+        refreshModelMgmtUI();
+        // Optionally, refresh when switching to the tab
+        document.getElementById('tab-models').addEventListener('click', refreshModelMgmtUI);
+
+        // Chat logic (streaming with OpenAI JS client placeholder)
+        const chatHistory = document.getElementById('chat-history');
+        const chatInput = document.getElementById('chat-input');
+        const sendBtn = document.getElementById('send-btn');
+        const modelSelect = document.getElementById('model-select');
+        let messages = [];
+
+        function appendMessage(role, text) {{
+            const div = document.createElement('div');
+            div.className = 'chat-message ' + role;
+            // Add a bubble for iMessage style
+            const bubble = document.createElement('div');
+            bubble.className = 'chat-bubble ' + role;
+            bubble.innerHTML = text;
+            div.appendChild(bubble);
+            chatHistory.appendChild(div);
+            chatHistory.scrollTop = chatHistory.scrollHeight;
+        }}
+
+        async function sendMessage() {{
+            const text = chatInput.value.trim();
+            if (!text) return;
+            appendMessage('user', text);
+            messages.push({{ role: 'user', content: text }});
+            chatInput.value = '';
+            sendBtn.disabled = true;
+            // Streaming OpenAI completions (placeholder, adapt as needed)
+            let llmText = '';
+            appendMessage('llm', '...');
+            const llmDiv = chatHistory.lastChild.querySelector('.chat-bubble.llm');
+            try {{
+                // Use the correct endpoint for chat completions
+                const resp = await fetch(getServerBaseUrl() + '/api/v1/chat/completions', {{
+                    method: 'POST',
+                    headers: {{ 'Content-Type': 'application/json' }},
+                    body: JSON.stringify({{
+                        model: modelSelect.value,
+                        messages: messages,
+                        stream: true
+                    }})
+                }});
+                if (!resp.body) throw new Error('No stream');
+                const reader = resp.body.getReader();
+                let decoder = new TextDecoder();
+                llmDiv.textContent = '';
+                while (true) {{
+                    const {{ done, value }} = await reader.read();
+                    if (done) break;
+                    const chunk = decoder.decode(value);
+                    if (chunk.trim() === 'data: [DONE]' || chunk.trim() === '[DONE]') continue;
+                    // Try to extract the content from the OpenAI chunk
+                    const match = chunk.match(/"content"\s*:\s*"([^"]*)"/);
+                    if (match && match[1]) {{
+                        llmText += match[1];
+                        llmDiv.textContent = llmText;
+                    }}
+                }}
+                messages.push({{ role: 'assistant', content: llmText }});
+            }} catch (e) {{
+                llmDiv.textContent = '[Error: ' + e.message + ']';
+            }}
+            sendBtn.disabled = false;
+        }}
+        sendBtn.onclick = sendMessage;
+        chatInput.addEventListener('keydown', function(e) {{
+            if (e.key === 'Enter') sendMessage();
+        }});
+        </script>
+    </body>
+    </html>
+    """
+    return HTMLResponse(content=styled_html)

lemonade_sdk-7.0.1/src/lemonade/tools/server/llamacpp.py

@@ -0,0 +1,289 @@
+import sys
+import os
+import logging
+import time
+import subprocess
+import zipfile
+import re
+import threading
+
+import requests
+from tabulate import tabulate
+from fastapi import HTTPException, status
+from fastapi.responses import StreamingResponse
+
+from openai import OpenAI
+
+from lemonade_server.model_manager import ModelManager
+from lemonade.tools.server.pydantic_models import ChatCompletionRequest
+
+LLAMA_VERSION = "b5543"
+LLAMA_SERVER_PORT = "8081"
+
+LLAMA_SERVER_EXE_DIR = os.path.join(
+    os.path.dirname(sys.executable),
+    "llama_server",
+)
+
+LLAMA_SERVER_EXE_PATH = os.path.join(
+    LLAMA_SERVER_EXE_DIR,
+    "llama-server.exe",
+)
+
+
+class LlamaTelemetry:
+    """
+    Manages telemetry data collection and display for llama server.
+    """
+
+    def __init__(self):
+        self.input_tokens = None
+        self.output_tokens = None
+        self.time_to_first_token = None
+        self.tokens_per_second = None
+        self.prompt_eval_time = None
+        self.eval_time = None
+
+    def parse_telemetry_line(self, line: str):
+        """
+        Parse telemetry data from llama server output lines.
+        """
+
+        # Parse prompt evaluation line
+        prompt_match = re.search(
+            # pylint: disable=C0301
+            r"prompt eval time\s*=\s*([\d.]+)\s*ms\s*/\s*(\d+)\s*tokens.*?([\d.]+)\s*tokens per second",
+            line,
+        )
+        if prompt_match:
+            prompt_time_ms = float(prompt_match.group(1))
+            input_tokens = int(prompt_match.group(2))
+
+            self.prompt_eval_time = prompt_time_ms / 1000.0
+            self.input_tokens = input_tokens
+            self.time_to_first_token = prompt_time_ms / 1000.0
+            return
+
+        # Parse generation evaluation line
+        eval_match = re.search(
+            r"eval time\s*=\s*([\d.]+)\s*ms\s*/\s*(\d+)\s*tokens.*?([\d.]+)\s*tokens per second",
+            line,
+        )
+        if eval_match:
+            eval_time_ms = float(eval_match.group(1))
+            output_tokens = int(eval_match.group(2))
+            tokens_per_second = float(eval_match.group(3))
+
+            self.eval_time = eval_time_ms / 1000.0
+            self.output_tokens = output_tokens
+            self.tokens_per_second = tokens_per_second
+            return
+
+    def get_telemetry_data(self):
+        return {
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "time_to_first_token": self.time_to_first_token,
+            "tokens_per_second": self.tokens_per_second,
+            "decode_token_times": None,
+        }
+
+    def show_telemetry(self):
+        # Check if debug logging is enabled
+        if not logging.getLogger().isEnabledFor(logging.DEBUG):
+            return
+
+        # Prepare telemetry data (transposed format)
+        telemetry = [
+            ["Input tokens", self.input_tokens],
+            ["Output tokens", self.output_tokens],
+            ["TTFT (s)", f"{self.time_to_first_token:.2f}"],
+            ["TPS", f"{self.tokens_per_second:.2f}"],
+        ]
+
+        table = tabulate(
+            telemetry, headers=["Metric", "Value"], tablefmt="fancy_grid"
+        ).split("\n")
+
+        # Show telemetry in debug while complying with uvicorn's log indentation
+        logging.debug("\n          ".join(table))
+
+
+def _log_subprocess_output(
+    process: subprocess.Popen, prefix: str, telemetry: LlamaTelemetry
+):
+    """
+    Read subprocess output line by line, log to debug, and parse telemetry
+    """
+
+    if process.stdout:
+        for line in iter(process.stdout.readline, ""):
+            if line:
+                line_stripped = line.strip()
+                logging.debug("%s: %s", prefix, line_stripped)
+
+                telemetry.parse_telemetry_line(line_stripped)
+
+            if process.poll() is not None:
+                break
+
+
+def _wait_for_load(llama_server_process: subprocess.Popen, fail_message: str):
+    status_code = None
+    while not llama_server_process.poll() and status_code != 200:
+        health_url = f"http://localhost:{LLAMA_SERVER_PORT}/health"
+        try:
+            health_response = requests.get(health_url)
+        except requests.exceptions.ConnectionError:
+            logging.warning(fail_message)
+        else:
+            status_code = health_response.status_code
+            logging.debug(
+                "Testing llama-server readiness (will retry until ready), "
+                f"result: {health_response.json()}"
+            )
+        time.sleep(1)
+
+
+def _launch_llama_subprocess(
+    model_path: str, use_gpu: bool, telemetry: LlamaTelemetry
+) -> subprocess.Popen:
+    """
+    Launch llama server subprocess with GPU or CPU configuration
+    """
+
+    base_command = [
+        LLAMA_SERVER_EXE_PATH,
+        "-m",
+        model_path,
+        "--port",
+        LLAMA_SERVER_PORT,
+    ]
+
+    # Configure GPU layers: 99 for GPU, 0 for CPU-only
+    ngl_value = "99" if use_gpu else "0"
+    command = base_command + ["-ngl", ngl_value]
+
+    # Start subprocess with output capture
+    process = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
+    )
+
+    # Start background thread to log subprocess output
+    device_type = "GPU" if use_gpu else "CPU"
+    threading.Thread(
+        target=_log_subprocess_output,
+        args=(process, f"LLAMA SERVER {device_type}", telemetry),
+        daemon=True,
+    ).start()
+
+    return process
+
+
+def server_load(checkpoint: str, model_reference: str, telemetry: LlamaTelemetry):
+    # Download llama.cpp server if it isn't already available
+    if not os.path.exists(LLAMA_SERVER_EXE_DIR):
+        # Download llama.cpp server zip
+        # pylint: disable=C0301
+        llama_zip_url = f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_VERSION}/llama-{LLAMA_VERSION}-bin-win-vulkan-x64.zip"
+        llama_zip_path = os.path.join(
+            os.path.dirname(sys.executable), "llama-server.zip"
+        )
+        logging.info(f"Downloading llama.cpp server from {llama_zip_url}")
+
+        with requests.get(llama_zip_url, stream=True) as r:
+            r.raise_for_status()
+            with open(llama_zip_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+
+        # Extract zip
+        logging.info(f"Extracting {llama_zip_path} to {LLAMA_SERVER_EXE_DIR}")
+        with zipfile.ZipFile(llama_zip_path, "r") as zip_ref:
+            zip_ref.extractall(LLAMA_SERVER_EXE_DIR)
+
+        # Save version.txt
+        version_txt_path = os.path.join(LLAMA_SERVER_EXE_DIR, "version.txt")
+        with open(version_txt_path, "w", encoding="utf-8") as vf:
+            vf.write(LLAMA_VERSION)
+
+        # Delete zip file
+        os.remove(llama_zip_path)
+        logging.info("Cleaned up zip file")
+
+    # Download the gguf to the hugging face cache
+    snapshot_path = ModelManager().download_gguf(checkpoint)
+    model_path = os.path.join(snapshot_path, os.listdir(snapshot_path)[0])
+    logging.debug(f"GGUF file path: {model_path}")
+
+    # Start the llama-serve.exe process
+    logging.debug(f"Using llama_server for GGUF model: {LLAMA_SERVER_EXE_PATH}")
+
+    # Attempt loading on GPU first
+    llama_server_process = _launch_llama_subprocess(
+        model_path, use_gpu=True, telemetry=telemetry
+    )
+
+    # Check the /health endpoint until GPU server is ready
+    _wait_for_load(
+        llama_server_process,
+        f"Loading {model_reference} on GPU didn't work, re-attempting on CPU",
+    )
+
+    # If loading on GPU failed, try loading on CPU
+    if llama_server_process.poll():
+        llama_server_process = _launch_llama_subprocess(
+            model_path, use_gpu=False, telemetry=telemetry
+        )
+
+        # Check the /health endpoint until CPU server is ready
+        _wait_for_load(
+            llama_server_process,
+            f"Loading {model_reference} on CPU didn't work",
+        )
+
+    if llama_server_process.poll():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Failed to load {model_reference} with llama.cpp",
+        )
+
+    return llama_server_process
+
+
+def chat_completion(
+    chat_completion_request: ChatCompletionRequest, telemetry: LlamaTelemetry
+):
+    base_url = f"http://127.0.0.1:{LLAMA_SERVER_PORT}/v1"
+    client = OpenAI(
+        base_url=base_url,
+        api_key="lemonade",
+    )
+
+    # Convert Pydantic model to dict and remove unset/null values
+    request_dict = chat_completion_request.model_dump(
+        exclude_unset=True, exclude_none=True
+    )
+
+    def event_stream():
+        try:
+            # Enable streaming
+            request_dict["stream"] = True
+            for chunk in client.chat.completions.create(**request_dict):
+                yield f"data: {chunk.model_dump_json()}\n\n"
+            yield "data: [DONE]\n\n"
+
+            # Show telemetry after completion
+            telemetry.show_telemetry()
+
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            yield f'data: {{"error": "{str(e)}"}}\n\n'
+
+    return StreamingResponse(
+        event_stream(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+        },
+    )