llms-py 3.0.0b6__py3-none-any.whl → 3.0.0b7__py3-none-any.whl

llms/{providers → extensions/providers}/google.py

@@ -1,5 +1,8 @@
+import base64
+import io
 import json
 import time
+import wave
 
 import aiohttp
 
@@ -11,7 +14,7 @@ import aiohttp
 #         self.chat_url = "https://generativelanguage.googleapis.com/v1beta/chat/completions"
 
 
-def install(ctx):
+def install_google(ctx):
     from llms.main import OpenAiCompatible
 
     def gemini_chat_summary(gemini_chat):
@@ -68,6 +71,7 @@ def install(ctx):
             super().__init__(**new_kwargs)
             self.safety_settings = kwargs.get("safety_settings")
             self.thinking_config = kwargs.get("thinking_config")
+            self.speech_config = kwargs.get("speech_config")
             self.tools = kwargs.get("tools")
             self.curl = kwargs.get("curl")
             self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
@@ -189,13 +193,15 @@ def install(ctx):
                     gemini_chat["generationConfig"] = generation_config
 
                 if "tools" in chat:
-                    gemini_chat["tools"] = chat["tools"]
+                    # gemini_chat["tools"] = chat["tools"]
+                    ctx.log("Error: tools not supported in Gemini")
                 elif self.tools:
-                    gemini_chat["tools"] = self.tools.copy()
+                    # gemini_chat["tools"] = self.tools.copy()
+                    ctx.log("Error: tools not supported in Gemini")
 
                 if "modalities" in chat:
                     generation_config["responseModalities"] = [modality.upper() for modality in chat["modalities"]]
-                    if "image_config" in chat:
+                    if "image" in chat["modalities"] and "image_config" in chat:
                         # delete thinkingConfig
                         del generation_config["thinkingConfig"]
                         config_map = {
@@ -205,6 +211,11 @@ def install(ctx):
                         generation_config["imageConfig"] = {
                             config_map[k]: v for k, v in chat["image_config"].items() if k in config_map
                         }
+                    if "audio" in chat["modalities"] and self.speech_config:
+                        del generation_config["thinkingConfig"]
+                        generation_config["speechConfig"] = self.speech_config.copy()
+                        # Currently Google Audio Models only accept AUDIO
+                        generation_config["responseModalities"] = ["AUDIO"]
 
                 started_at = int(time.time() * 1000)
                 gemini_chat_url = f"https://generativelanguage.googleapis.com/v1beta/models/{chat['model']}:generateContent?key={self.api_key}"
@@ -218,13 +229,22 @@ def install(ctx):
                     with open(f"{ctx.MOCK_DIR}/gemini-image.json") as f:
                         obj = json.load(f)
                 else:
-                    async with session.post(
-                        gemini_chat_url,
-                        headers=self.headers,
-                        data=json.dumps(gemini_chat),
-                        timeout=aiohttp.ClientTimeout(total=120),
-                    ) as res:
-                        obj = await self.response_json(res)
+                    try:
+                        async with session.post(
+                            gemini_chat_url,
+                            headers=self.headers,
+                            data=json.dumps(gemini_chat),
+                            timeout=aiohttp.ClientTimeout(total=120),
+                        ) as res:
+                            obj = await self.response_json(res)
+                    except Exception as e:
+                        ctx.log(f"Error: {res.status} {res.reason}: {e}")
+                        text = await res.text()
+                        try:
+                            obj = json.loads(text)
+                        except:
+                            ctx.log(text)
+                            raise e
 
                 if "error" in obj:
                     ctx.log(f"Error: {obj['error']}")
@@ -233,6 +253,18 @@ def install(ctx):
                 if ctx.debug:
                     ctx.dbg(json.dumps(gemini_response_summary(obj), indent=2))
 
+                # calculate cost per generation
+                cost = None
+                token_costs = obj.get("metadata", {}).get("pricing", "")
+                if token_costs:
+                    input_price, output_price = token_costs.split("/")
+                    input_per_token = float(input_price) / 1000000
+                    output_per_token = float(output_price) / 1000000
+                    if "usageMetadata" in obj:
+                        input_tokens = obj["usageMetadata"].get("promptTokenCount", 0)
+                        output_tokens = obj["usageMetadata"].get("candidatesTokenCount", 0)
+                        cost = (input_per_token * input_tokens) + (output_per_token * output_tokens)
+
                 response = {
                     "id": f"chatcmpl-{started_at}",
                     "created": started_at,
@@ -248,6 +280,7 @@ def install(ctx):
                     content = ""
                     reasoning = ""
                     images = []
+                    audios = []
                     if "content" in candidate and "parts" in candidate["content"]:
                         text_parts = []
                         reasoning_parts = []
@@ -260,19 +293,59 @@ def install(ctx):
                             if "inlineData" in part:
                                 inline_data = part["inlineData"]
                                 mime_type = inline_data.get("mimeType", "image/png")
-                                ext = mime_type.split("/")[1]
-                                base64_data = inline_data["data"]
-                                filename = f"{chat['model'].split('/')[-1]}-{len(images)}.{ext}"
-                                relative_url, info = ctx.save_image_to_cache(base64_data, filename, {})
-                                images.append(
-                                    {
-                                        "type": "image_url",
-                                        "index": len(images),
-                                        "image_url": {
-                                            "url": relative_url,
-                                        },
-                                    }
-                                )
+                                if mime_type.startswith("image"):
+                                    ext = mime_type.split("/")[1]
+                                    base64_data = inline_data["data"]
+                                    filename = f"{chat['model'].split('/')[-1]}-{len(images)}.{ext}"
+                                    ctx.log(f"inlineData {len(base64_data)} {mime_type} {filename}")
+                                    relative_url, info = ctx.save_image_to_cache(
+                                        base64_data,
+                                        filename,
+                                        ctx.to_file_info(chat, {"cost": cost}),
+                                    )
+                                    images.append(
+                                        {
+                                            "type": "image_url",
+                                            "index": len(images),
+                                            "image_url": {
+                                                "url": relative_url,
+                                            },
+                                        }
+                                    )
+                                elif mime_type.startswith("audio"):
+                                    # mime_type audio/L16;codec=pcm;rate=24000
+                                    base64_data = inline_data["data"]
+
+                                    pcm = base64.b64decode(base64_data)
+                                    # Convert PCM to WAV
+                                    wav_io = io.BytesIO()
+                                    with wave.open(wav_io, "wb") as wf:
+                                        wf.setnchannels(1)
+                                        wf.setsampwidth(2)
+                                        wf.setframerate(24000)
+                                        wf.writeframes(pcm)
+                                    wav_data = wav_io.getvalue()
+
+                                    ext = mime_type.split("/")[1].split(";")[0]
+                                    pcm_filename = f"{chat['model'].split('/')[-1]}-{len(audios)}.{ext}"
+                                    filename = pcm_filename.replace(f".{ext}", ".wav")
+                                    ctx.log(f"inlineData {len(base64_data)} {mime_type} {filename}")
+
+                                    relative_url, info = ctx.save_bytes_to_cache(
+                                        wav_data,
+                                        filename,
+                                        ctx.to_file_info(chat, {"cost": cost}),
+                                    )
+
+                                    audios.append(
+                                        {
+                                            "type": "audio_url",
+                                            "index": len(audios),
+                                            "audio_url": {
+                                                "url": relative_url,
+                                            },
+                                        }
+                                    )
                         content = " ".join(text_parts)
                         reasoning = " ".join(reasoning_parts)
 
@@ -288,6 +361,8 @@ def install(ctx):
                         choice["message"]["reasoning"] = reasoning
                     if len(images) > 0:
                         choice["message"]["images"] = images
+                    if len(audios) > 0:
+                        choice["message"]["audios"] = audios
                     choices.append(choice)
                 response["choices"] = choices
                 if "usageMetadata" in obj:
@@ -301,6 +376,3 @@ def install(ctx):
                 return ctx.log_json(self.to_response(response, chat, started_at))
 
     ctx.add_provider(GoogleProvider)
-
-
-__install__ = install

llms/{providers → extensions/providers}/nvidia.py

@@ -4,7 +4,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_nvidia(ctx):
     from llms.main import GeneratorBase
 
     class NvidiaGenAi(GeneratorBase):
@@ -29,10 +29,11 @@ def install(ctx):
                         last_model = "/" in chat["model"] and chat["model"].split("/")[-1] or chat["model"]
                         filename = f"{last_model}_{seed}.png"
 
-                    image_info = {
-                        "seed": seed,
-                    }
-                    relative_url, info = ctx.save_image_to_cache(base64, filename, image_info)
+                    relative_url, info = ctx.save_image_to_cache(
+                        base64,
+                        filename,
+                        ctx.to_file_info(chat, {"seed": seed}),
+                    )
                     return {
                         "choices": [
                             {
@@ -102,6 +103,3 @@ def install(ctx):
                     return self.to_response(await self.response_json(response), chat, started_at)
 
     ctx.add_provider(NvidiaGenAi)
-
-
-__install__ = install

llms/{providers → extensions/providers}/openai.py

@@ -6,7 +6,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_openai(ctx):
     from llms.main import GeneratorBase, OpenAiCompatible
 
     class OpenAiProvider(OpenAiCompatible):
@@ -83,10 +83,7 @@ def install(ctx):
                         relative_url, info = ctx.save_image_to_cache(
                             image_data,
                             f"{chat['model']}-{i}.{ext}",
-                            {
-                                "model": chat["model"],
-                                "prompt": ctx.last_user_prompt(chat),
-                            },
+                            ctx.to_file_info(chat),
                         )
                         images.append(
                             {
@@ -156,4 +153,4 @@ def install(ctx):
     ctx.add_provider(OpenAiGenerator)
 
 
-__install__ = install
+__install__ = install_openai

llms/{providers → extensions/providers}/openrouter.py

@@ -4,7 +4,7 @@ import time
 import aiohttp
 
 
-def install(ctx):
+def install_openrouter(ctx):
     from llms.main import GeneratorBase
 
     # https://openrouter.ai/docs/guides/overview/multimodal/image-generation
@@ -16,6 +16,9 @@ def install(ctx):
 
         def to_response(self, response, chat, started_at):
             # go through all image responses and save them to cache
+            cost = None
+            if "usage" in response and "cost" in response["usage"]:
+                cost = response["usage"]["cost"]
             for choice in response["choices"]:
                 if "message" in choice and "images" in choice["message"]:
                     for image in choice["message"]["images"]:
@@ -29,11 +32,9 @@ def install(ctx):
                                 base64_data = parts[1]
                                 model = chat["model"].split("/")[-1]
                                 filename = f"{model}-{choice['index']}.{ext}"
-                                info = {
-                                    "model": model,
-                                    "prompt": ctx.last_user_prompt(chat),
-                                }
-                                relative_url, info = ctx.save_image_to_cache(base64_data, filename, info)
+                                relative_url, info = ctx.save_image_to_cache(
+                                    base64_data, filename, ctx.to_file_info(chat, {"cost": cost})
+                                )
                                 image["image_url"]["url"] = relative_url
 
             return response
@@ -50,11 +51,13 @@ def install(ctx):
                 return ctx.log_json(self.to_response(json.loads(text), chat, started_at))
             else:
                 chat_url = provider.chat_url
+                # remove tools
+                chat.pop("tools", None)
                 chat = await self.process_chat(chat, provider_id=self.id)
                 ctx.log(f"POST {chat_url}")
                 ctx.log(provider.chat_summary(chat))
                 # remove metadata if any (conflicts with some providers, e.g. Z.ai)
-                chat.pop("metadata", None)
+                metadata = chat.pop("metadata", None)
 
                 async with aiohttp.ClientSession() as session, session.post(
                     chat_url,
@@ -62,9 +65,8 @@ def install(ctx):
                     data=json.dumps(chat),
                     timeout=aiohttp.ClientTimeout(total=300),
                 ) as response:
+                    if metadata:
+                        chat["metadata"] = metadata
                     return ctx.log_json(self.to_response(await self.response_json(response), chat, started_at))
 
     ctx.add_provider(OpenRouterGenerator)
-
-
-__install__ = install

llms/extensions/system_prompts/__init__.py

@@ -0,0 +1,45 @@
+import json
+import os
+
+from aiohttp import web
+
+default_prompts = [
+    {"name": "Helpful Assistant", "prompt": "You are a helpful assistant."},
+]
+
+
+# runs after providers are configured but before server is run
+def install(ctx):
+    # helper to get user or default prompts
+    def get_user_prompts(request):
+        candidate_paths = []
+        # check if user is signed in
+        username = ctx.get_username(request)
+        if username:
+            # if signed in (Github OAuth), return the prompts for this user if exists
+            candidate_paths.append(os.path.join(ctx.get_user_path(username), "system_prompts", "prompts.json"))
+        # return default prompts for all users if exists
+        candidate_paths.append(os.path.join(ctx.get_user_path(), "system_prompts", "prompts.json"))
+        # otherwise return the default prompts from this repo
+        candidate_paths.append(os.path.join(ctx.path, "ui", "prompts.json"))
+
+        # iterate all candidate paths and when exists return its json
+        for path in candidate_paths:
+            if os.path.exists(path):
+                with open(path, encoding="utf-8") as f:
+                    txt = f.read()
+                    return json.loads(txt)
+        return default_prompts
+
+    # API Handler to get prompts
+    async def get_prompts(request):
+        prompts_json = get_user_prompts(request)
+        return web.json_response(prompts_json)
+
+    ctx.add_get("prompts.json", get_prompts)
+
+
+# register install extension handler
+__install__ = install
+
+__order__ = -10

llms/extensions/system_prompts/ui/index.mjs

@@ -0,0 +1,284 @@
+import { ref, computed, inject, watch, onMounted, onUnmounted, nextTick } from "vue"
+import { AppContext } from "ctx.mjs"
+
+let ext
+
+const PromptFinder = {
+    template: `
+    <div v-if="modelValue" class="absolute right-0 top-full z-10 mt-1 origin-top-right rounded-md bg-white dark:bg-gray-900 shadow-lg border border-gray-300 dark:border-gray-600 focus:outline-none"
+         style="width:400px"
+         role="menu" aria-orientation="vertical" aria-labelledby="menu-button" tabindex="-1">
+        <div class="p-2" role="none">
+            <div class="relative mb-2">
+                <div class="pointer-events-none absolute inset-y-0 left-0 flex items-center pl-3">
+                    <svg class="h-4 w-4 text-gray-400" viewBox="0 0 20 20" fill="currentColor" aria-hidden="true">
+                        <path fill-rule="evenodd" d="M9 3.5a5.5 5.5 0 100 11 5.5 5.5 0 000-11zM2 9a7 7 0 1112.452 4.391l3.328 3.329a.75.75 0 11-1.06 1.06l-3.329-3.328A7 7 0 012 9z" clip-rule="evenodd" />
+                    </svg>
+                </div>
+                <input type="text" 
+                    ref="searchInput"
+                    v-model="searchQuery"
+                    @keydown="onKeydown"
+                    class="block w-full rounded-md border-0 py-1.5 pl-10 text-gray-900 dark:text-gray-100 shadow-sm ring-1 ring-inset ring-gray-300 dark:ring-gray-600 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-blue-600 sm:text-xs sm:leading-6 bg-transparent" 
+                    placeholder="Search prompts...">
+            </div>
+
+            <div class="max-h-80 overflow-y-auto" ref="resultsList">
+                <div v-if="filteredPrompts.length === 0" class="p-4 text-center text-xs text-gray-500">
+                    No prompts found
+                </div>
+                <div v-for="(prompt, index) in filteredPrompts" :key="prompt.id" 
+                    @click="selectPrompt(prompt)"
+                    :class="['group relative flex gap-x-2 rounded-md p-2 cursor-pointer border-b border-gray-100 dark:border-gray-800 last:border-0', 
+                             selectedIndex === index ? 'bg-blue-50 dark:bg-blue-900/20' : 'hover:bg-gray-50 dark:hover:bg-gray-800']"
+                    :data-index="index">
+                    <div class="flex-auto">
+                        <div class="flex items-center justify-between">
+                            <h4 :class="['font-semibold text-sm', selectedIndex === index ? 'text-blue-700 dark:text-blue-300' : 'text-gray-900 dark:text-gray-100']">
+                                {{ prompt.name }}
+                            </h4>
+                        </div>
+                        <p class="text-xs leading-4 text-gray-500 dark:text-gray-400 line-clamp-2 mt-0.5">{{ prompt.value }}</p>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+    `,
+    props: {
+        modelValue: Boolean, // controls visibility
+        prompts: {
+            type: Array,
+            default: () => []
+        }
+    },
+    emits: ['update:modelValue', 'select'],
+    setup(props, { emit }) {
+        const searchQuery = ref('')
+        const searchInput = ref(null)
+        const resultsList = ref(null)
+        const selectedIndex = ref(-1)
+
+        const filteredPrompts = computed(() => {
+            if (!searchQuery.value) return props.prompts
+            const q = searchQuery.value.toLowerCase()
+            return props.prompts.filter(p =>
+                p.name.toLowerCase().includes(q) ||
+                p.value.toLowerCase().includes(q) ||
+                p.id.toLowerCase().includes(q)
+            )
+        })
+
+        function selectPrompt(prompt) {
+            emit('select', prompt)
+            emit('update:modelValue', false)
+        }
+
+        function scrollToSelected() {
+            nextTick(() => {
+                if (!resultsList.value) return
+                const el = resultsList.value.querySelector(`[data-index="${selectedIndex.value}"]`)
+                if (el) {
+                    el.scrollIntoView({ block: 'nearest' })
+                }
+            })
+        }
+
+        function onKeydown(e) {
+            if (filteredPrompts.value.length === 0) return
+
+            if (e.key === 'ArrowDown') {
+                e.preventDefault()
+                selectedIndex.value = (selectedIndex.value + 1) % filteredPrompts.value.length
+                scrollToSelected()
+            } else if (e.key === 'ArrowUp') {
+                e.preventDefault()
+                selectedIndex.value = (selectedIndex.value - 1 + filteredPrompts.value.length) % filteredPrompts.value.length
+                scrollToSelected()
+            } else if (e.key === 'Enter') {
+                e.preventDefault()
+                if (selectedIndex.value >= 0 && selectedIndex.value < filteredPrompts.value.length) {
+                    selectPrompt(filteredPrompts.value[selectedIndex.value])
+                }
+            }
+        }
+
+        watch(() => props.modelValue, (isOpen) => {
+            if (isOpen) {
+                // Focus search input when modal opens
+                nextTick(() => {
+                    if (searchInput.value) {
+                        searchInput.value.focus()
+                    }
+                })
+                selectedIndex.value = -1
+            } else {
+                searchQuery.value = ''
+            }
+        })
+
+        watch(searchQuery, () => {
+            selectedIndex.value = 0 // Select first result on search
+        })
+
+        return {
+            searchQuery,
+            searchInput,
+            resultsList,
+            filteredPrompts,
+            selectedIndex,
+            selectPrompt,
+            onKeydown
+        }
+    }
+}
+
+const SystemPromptEditor = {
+    template: `
+    <div class="border-b border-gray-200 dark:border-gray-700 px-6 pb-4">
+        <div class="max-w-6xl mx-auto">
+            <div class="mt-2 h-10 flex justify-between items-center">
+                <label class="select-none block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
+                    System Prompt
+                </label>
+                <div v-if="hasMessages" class="text-sm text-gray-500 dark:text-gray-400">
+                    {{ !threadSystemPrompt ? '' : prompts.find(x => x.value === threadSystemPrompt)?.name || 'Custom' }}
+                </div>
+                <div v-else class="mb-2 relative" ref="containerRef">
+                    <div class="flex items-center gap-2">
+                        <span v-if="selected" class="text-sm text-gray-500 dark:text-gray-400">
+                            {{ selected.name }}
+                        </span>
+                        <button v-if="modelValue" type="button" title="Clear System Prompt" @click="$emit('update:modelValue', null)"
+                            class="rounded-full p-1 hover:bg-gray-200 dark:hover:bg-gray-700 transition-colors">
+                            <svg class="size-4 text-gray-500 dark:text-gray-400" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path fill="currentColor" d="M19 6.41L17.59 5L12 10.59L6.41 5L5 6.41L10.59 12L5 17.59L6.41 19L12 13.41L17.59 19L19 17.59L13.41 12z"/></svg>
+                        </button>
+                        <button type="button" 
+                            @click="showFinder = !showFinder"
+                            class="inline-flex items-center gap-x-1.5 rounded-md bg-white dark:bg-gray-900 px-2.5 py-1.5 text-sm font-medium text-gray-700 dark:text-gray-300 shadow-sm border border-gray-300 dark:border-gray-600 hover:bg-gray-50 dark:hover:bg-gray-800">
+                            Explore Prompts
+                        </button>
+                    </div>
+                    <PromptFinder v-model="showFinder" :prompts="prompts" @select="onSelect" />
+                </div>
+            </div>
+            <div v-if="hasMessages" class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-900 text-gray-900 dark:text-gray-100 px-3 py-2 text-sm">
+                {{threadSystemPrompt || 'No System Prompt was used' }}
+            </div>
+            <div v-else>
+                <textarea
+                    :value="modelValue" @input="$emit('update:modelValue', $event.target.value)"
+                    placeholder="Enter a system prompt to guide AI's behavior..."
+                    rows="6"
+                    class="block w-full resize-vertical rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-900 text-gray-900 dark:text-gray-100 px-3 py-2 text-sm placeholder-gray-500 dark:placeholder-gray-400 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500"
+                ></textarea>
+            </div>
+        </div>
+    </div>
+    `,
+    emits: ['update:modelValue'],
+    props: {
+        prompts: Array,
+        selected: Object,
+        modelValue: String,
+    },
+    setup(props, { emit }) {
+        /**@type {AppContext} */
+        const ctx = inject('ctx')
+        const containerRef = ref()
+        const showFinder = ref(false)
+        const prefs = ext.getPrefs()
+        const hasMessages = computed(() => ctx.threads.currentThread.value?.messages?.length > 0)
+        const threadSystemPrompt = computed(() => ctx.threads.currentThread.value?.systemPrompt || '')
+        const selected = computed(() =>
+            props.prompts.find(x => x.value === props.modelValue) ?? { name: "Custom", value: props.modelValue })
+
+        function onSelect(prompt) {
+            emit('update:modelValue', prompt.value)
+        }
+
+        function closeFinder(e) {
+            if (showFinder.value && containerRef.value && !containerRef.value.contains(e.target)) {
+                showFinder.value = false
+            }
+        }
+
+        watch(() => props.modelValue, promptValue => {
+            prefs.prompt = selected.value
+            ext.setPrefs(prefs)
+        })
+
+        onMounted(() => {
+            document.addEventListener('click', closeFinder)
+            if (prefs.prompt) {
+                emit('update:modelValue', prefs.prompt.value)
+            }
+        })
+        onUnmounted(() => {
+            document.removeEventListener('click', closeFinder)
+        })
+
+        return {
+            threadSystemPrompt,
+            hasMessages,
+            selected,
+            containerRef,
+            showFinder,
+            onSelect,
+        }
+    }
+}
+
+export default {
+    order: 30 - 100,
+
+    install(ctx) {
+        ext = ctx.scope('system_prompts')
+        ctx.components({
+            PromptFinder,
+            SystemPromptEditor,
+            SystemPromptsPanel: {
+                template: `<SystemPromptEditor :prompts="$state.prompts" v-model="$state.selectedPrompt" />`,
+            }
+        })
+
+        ctx.setTopIcons({
+            system_prompts: {
+                component: {
+                    template: `<svg @click="$ctx.toggleTop('SystemPromptsPanel')" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="m5 7l5 5l-5 5m8 0h6"/></svg>`,
+                },
+                isActive({ top }) { return top === 'SystemPromptsPanel' }
+            }
+        })
+
+        ctx.createThreadFilters.push(thread => {
+            const prefs = ext.getPrefs()
+            thread.systemPrompt = prefs?.prompt?.value || ""
+            console.log('createThreadFilters', prefs, thread)
+        })
+
+        ctx.chatRequestFilters.push(({ request, thread }) => {
+
+            const hasSystemPrompt = request.messages.find(x => x.role === 'system')
+            if (hasSystemPrompt) {
+                console.log('Already has system prompt', hasSystemPrompt.content)
+                return
+            }
+
+            if (thread.systemPrompt) {
+                // add message to start
+                request.messages.unshift({
+                    role: 'system',
+                    content: thread.systemPrompt
+                })
+            }
+        })
+
+        ctx.setState({ prompts: [] })
+    },
+
+    async load(ctx) {
+        const prompts = await ext.getJson(`/prompts.json`)
+        ctx.setState({ prompts })
+    }
+}