threadshelf 1.2.0

package/dist/src/generation/model-download.js

@@ -0,0 +1,151 @@
+import { mkdir, rm, stat } from 'fs/promises';
+import { isAbsolute, join, relative, resolve, sep } from 'path';
+import { defaultDownloadDirectory, getGenerationConfig } from './config.js';
+import { downloadToFile, sha256File } from './downloader.js';
+import { assertRepoId, catalogFileUrl, getCatalogModel, huggingFaceToken, CatalogError, } from './model-catalog.js';
+/** Turns `unsloth/Qwen3.5-4B-GGUF` into a flat, filesystem-safe folder name. */
+export const modelFolderName = (repoId) => assertRepoId(repoId).replace(/\//g, '__').replace(/[^A-Za-z0-9._-]/g, '_');
+const assertInside = (root, candidate) => {
+    const absolute = resolve(candidate);
+    const rel = relative(resolve(root), absolute);
+    // `isAbsolute` catches the cross-drive case, where `relative` returns a full
+    // path that has no `..` prefix to detect.
+    if (isAbsolute(rel) ||
+        rel === '' ||
+        rel === '..' ||
+        rel.startsWith(`..${sep}`) ||
+        rel.includes('\0')) {
+        throw new CatalogError('Refusing to write a model file outside the download directory', 400);
+    }
+    return absolute;
+};
+/**
+ * File names come from a remote API, so only a plain relative path of safe
+ * segments is accepted before it is joined onto the download directory.
+ */
+const safeRepoPath = (repoPath) => {
+    const segments = repoPath.split('/').filter(Boolean);
+    if (segments.length === 0 ||
+        segments.length > 4 ||
+        segments.some((segment) => segment === '.' || segment === '..' || !/^[A-Za-z0-9._-]{1,128}$/.test(segment))) {
+        throw new CatalogError(`Unsafe model file path: ${repoPath}`, 400);
+    }
+    return segments.join('/');
+};
+export const resolveDownloadDirectory = async () => {
+    const config = await getGenerationConfig().catch(() => null);
+    return config?.llamaCpp.downloadDirectory ?? defaultDownloadDirectory();
+};
+export const planModelDownload = async ({ repoId, quant, includeProjector = false, directory, fetchImpl = fetch, detail, }) => {
+    const id = assertRepoId(repoId);
+    const model = detail ?? (await getCatalogModel(id, fetchImpl));
+    const selected = quant
+        ? model.quants.find((entry) => entry.label.toUpperCase() === quant.toUpperCase())
+        : (model.quants.find((entry) => entry.recommended) ?? model.quants[0]);
+    if (!selected) {
+        throw new CatalogError(`${id} does not publish a "${quant}" quantisation`, 404);
+    }
+    const root = resolve(directory ?? (await resolveDownloadDirectory()));
+    const folder = join(root, modelFolderName(id));
+    const chosen = includeProjector
+        ? [...selected.files, ...model.projectors]
+        : selected.files;
+    const files = chosen.map((file) => {
+        const repoPath = safeRepoPath(file.path);
+        return {
+            url: catalogFileUrl(id, repoPath),
+            repoPath,
+            // Shards and projectors are flattened next to each other; llama.cpp finds
+            // sibling shards by name, so the repository's folder layout is not needed.
+            destination: assertInside(folder, join(folder, repoPath.split('/').pop())),
+            sizeBytes: file.sizeBytes,
+            sha256: file.sha256,
+        };
+    });
+    const primary = files.find((file) => !/^mmproj/i.test(file.repoPath.split('/').pop() ?? ''));
+    if (!primary)
+        throw new CatalogError('Selected quantisation has no model file', 404);
+    return {
+        repoId: id,
+        quant: selected.label,
+        files,
+        totalBytes: files.reduce((sum, file) => sum + file.sizeBytes, 0),
+        directory: folder,
+        primaryPath: primary.destination,
+        gated: model.gated,
+        license: model.license,
+        contextLength: model.contextLength,
+        requiresToken: model.gated !== false && !huggingFaceToken(),
+    };
+};
+const alreadyPresent = async (file) => {
+    const info = await stat(file.destination).catch(() => null);
+    if (!info?.isFile() || info.size !== file.sizeBytes)
+        return false;
+    if (!file.sha256)
+        return true;
+    return (await sha256File(file.destination)) === file.sha256;
+};
+/**
+ * Cheap "is this model already on disk?" check for planning screens: name and
+ * exact byte size only, no hashing. Re-reading 17 GB just to render a button
+ * label is not worth it — `downloadModel` still verifies the digest before it
+ * skips anything, so a size collision costs one re-download, not a bad model.
+ */
+export const modelFilesPresent = async (plan) => {
+    if (plan.files.length === 0)
+        return false;
+    for (const file of plan.files) {
+        const info = await stat(file.destination).catch(() => null);
+        if (!info?.isFile() || info.size !== file.sizeBytes)
+            return false;
+    }
+    return true;
+};
+export const downloadModel = async (plan, { onProgress, signal, } = {}) => {
+    if (plan.requiresToken) {
+        throw new CatalogError(`${plan.repoId} is a gated repository. Accept its licence on Hugging Face and set HF_TOKEN in the server .env.`, 403);
+    }
+    await mkdir(plan.directory, { recursive: true });
+    const token = huggingFaceToken();
+    let completedBytes = 0;
+    for (const [index, file] of plan.files.entries()) {
+        signal?.throwIfAborted();
+        onProgress?.({ phase: 'verifying', file: file.repoPath });
+        if (await alreadyPresent(file)) {
+            completedBytes += file.sizeBytes;
+            onProgress?.({ phase: 'skipped', file: file.repoPath });
+            continue;
+        }
+        const baseline = completedBytes;
+        try {
+            await downloadToFile(file.url, file.destination, {
+                sha256: file.sha256,
+                expectedBytes: file.sizeBytes || undefined,
+                signal,
+                headers: token ? { Authorization: `Bearer ${token}` } : {},
+                onProgress: (progress) => onProgress?.({
+                    phase: 'downloading',
+                    file: file.repoPath,
+                    fileIndex: index + 1,
+                    totalFiles: plan.files.length,
+                    downloadedBytes: baseline + progress.downloadedBytes,
+                    totalBytes: plan.totalBytes,
+                }),
+            });
+        }
+        catch (error) {
+            // Cancelling is not a fault: keep the partial file so pressing Download
+            // again resumes with a Range request instead of starting over. Any other
+            // failure drops it, because a partial shard that llama.cpp would try to
+            // load is worse than no shard at all.
+            if (!(error instanceof Error && error.name === 'AbortError')) {
+                await rm(`${file.destination}.part`, { force: true }).catch(() => undefined);
+            }
+            throw error;
+        }
+        completedBytes += file.sizeBytes;
+    }
+    onProgress?.({ phase: 'completed', primaryPath: plan.primaryPath });
+    return { primaryPath: plan.primaryPath, directory: plan.directory };
+};

package/dist/src/generation/openai-compatible.js

@@ -0,0 +1,231 @@
+const requestHeaders = (apiKey) => {
+    const headers = { 'Content-Type': 'application/json' };
+    if (apiKey)
+        headers.Authorization = `Bearer ${apiKey}`;
+    return headers;
+};
+const requestSignal = (provider, signal) => {
+    const timeout = AbortSignal.timeout(provider === 'llama-cpp' ? 30 * 60_000 : 5 * 60_000);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+};
+const errorChain = (error) => {
+    const messages = [];
+    const seen = new Set();
+    let current = error;
+    while (current && !seen.has(current) && messages.length < 5) {
+        seen.add(current);
+        if (current instanceof Error) {
+            if (current.message && !messages.includes(current.message))
+                messages.push(current.message);
+            current = current.cause;
+        }
+        else {
+            const message = String(current);
+            if (message && !messages.includes(message))
+                messages.push(message);
+            break;
+        }
+    }
+    return messages.join(' → ') || 'unknown transport error';
+};
+const providerRequestError = (provider, stage, error) => new Error(`${provider} ${stage}: ${errorChain(error)}`, { cause: error });
+const responseErrorMessage = async (response, provider) => {
+    const raw = await response.text().catch(() => '');
+    if (raw) {
+        try {
+            const payload = JSON.parse(raw);
+            if (payload.error?.message)
+                return payload.error.message;
+        }
+        catch {
+            const compact = raw.replace(/\s+/g, ' ').trim();
+            if (compact)
+                return compact.slice(0, 2_000);
+        }
+    }
+    return `${provider} request failed (${response.status} ${response.statusText || 'HTTP error'})`;
+};
+const requestBody = (request, stream, extraBody) => JSON.stringify({
+    model: request.model,
+    messages: request.messages,
+    temperature: request.temperature,
+    max_tokens: request.maxTokens,
+    stream,
+    ...(stream ? { stream_options: { include_usage: true } } : {}),
+    ...extraBody,
+});
+const textDelta = (value) => {
+    if (typeof value === 'string')
+        return value;
+    if (!Array.isArray(value))
+        return '';
+    return value
+        .map((part) => {
+        if (!part || typeof part !== 'object')
+            return '';
+        const text = part.text;
+        return typeof text === 'string' ? text : '';
+    })
+        .join('');
+};
+const responseUsage = (usage) => usage
+    ? {
+        promptTokens: usage.prompt_tokens,
+        completionTokens: usage.completion_tokens,
+        totalTokens: usage.total_tokens,
+    }
+    : undefined;
+const responsePerformance = (timings, usage, elapsedMs) => {
+    if (timings?.predicted_per_second && Number.isFinite(timings.predicted_per_second)) {
+        return {
+            completionTokensPerSecond: timings.predicted_per_second,
+            promptTokensPerSecond: timings.prompt_per_second,
+            generationMs: timings.predicted_ms,
+            source: 'provider',
+        };
+    }
+    if (!usage?.completion_tokens || elapsedMs <= 0)
+        return undefined;
+    return {
+        completionTokensPerSecond: usage.completion_tokens / (elapsedMs / 1000),
+        generationMs: elapsedMs,
+        source: 'measured',
+    };
+};
+export const openAiCompatibleChat = async ({ provider, baseUrl, apiKey, request, extraBody, signal, fetchImpl = fetch, }) => {
+    const startedAt = performance.now();
+    let response;
+    try {
+        response = await fetchImpl(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
+            method: 'POST',
+            headers: requestHeaders(apiKey),
+            body: requestBody(request, false, extraBody),
+            signal: requestSignal(provider, signal),
+        });
+    }
+    catch (error) {
+        throw providerRequestError(provider, 'connection failed', error);
+    }
+    if (!response.ok) {
+        throw new Error(`${provider} request failed: ${await responseErrorMessage(response, provider)}`);
+    }
+    const payload = (await response.json().catch(() => ({})));
+    const message = payload.choices?.[0]?.message;
+    const content = message?.content?.trim();
+    if (!content)
+        throw new Error(`${provider} returned an empty response`);
+    const reasoning = (message?.reasoning_content || message?.reasoning || '').trim() || undefined;
+    return {
+        provider,
+        model: payload.model || request.model,
+        content,
+        reasoning,
+        usage: responseUsage(payload.usage),
+        performance: responsePerformance(payload.timings, payload.usage, performance.now() - startedAt),
+    };
+};
+export const openAiCompatibleChatStream = async (options, onDelta) => {
+    const { provider, baseUrl, apiKey, request, extraBody, signal, fetchImpl = fetch } = options;
+    const startedAt = performance.now();
+    let response;
+    try {
+        response = await fetchImpl(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
+            method: 'POST',
+            headers: requestHeaders(apiKey),
+            body: requestBody(request, true, extraBody),
+            signal: requestSignal(provider, signal),
+        });
+    }
+    catch (error) {
+        throw providerRequestError(provider, 'connection failed', error);
+    }
+    if (!response.ok) {
+        throw new Error(`${provider} request failed: ${await responseErrorMessage(response, provider)}`);
+    }
+    if (!response.body)
+        throw new Error(`${provider} returned a response without a stream`);
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let content = '';
+    let reasoning = '';
+    let model = request.model;
+    let usage;
+    let timings;
+    let streamDone = false;
+    const consumeLine = async (rawLine) => {
+        const line = rawLine.trim();
+        if (!line.startsWith('data:'))
+            return;
+        const data = line.slice(5).trim();
+        if (!data)
+            return;
+        if (data === '[DONE]') {
+            streamDone = true;
+            return;
+        }
+        let chunk;
+        try {
+            chunk = JSON.parse(data);
+        }
+        catch (error) {
+            throw new Error(`${provider} returned an invalid streaming event`, { cause: error });
+        }
+        if (chunk.error)
+            throw new Error(chunk.error.message || `${provider} streaming request failed`);
+        if (chunk.model)
+            model = chunk.model;
+        if (chunk.usage)
+            usage = chunk.usage;
+        if (chunk.timings)
+            timings = chunk.timings;
+        const delta = chunk.choices?.[0]?.delta;
+        const contentPart = textDelta(delta?.content);
+        const reasoningPart = textDelta(delta?.reasoning_content ?? delta?.reasoning);
+        if (!contentPart && !reasoningPart)
+            return;
+        content += contentPart;
+        reasoning += reasoningPart;
+        await onDelta({
+            content: contentPart || undefined,
+            reasoning: reasoningPart || undefined,
+            model,
+        });
+    };
+    while (!streamDone) {
+        let result;
+        try {
+            result = await reader.read();
+        }
+        catch (error) {
+            throw providerRequestError(provider, 'response stream failed', error);
+        }
+        const { done, value } = result;
+        buffer += decoder.decode(value, { stream: !done });
+        let newline = buffer.indexOf('\n');
+        while (newline !== -1) {
+            const line = buffer.slice(0, newline).replace(/\r$/, '');
+            buffer = buffer.slice(newline + 1);
+            await consumeLine(line);
+            if (streamDone)
+                break;
+            newline = buffer.indexOf('\n');
+        }
+        if (done)
+            break;
+    }
+    if (!streamDone && buffer.trim())
+        await consumeLine(buffer);
+    if (streamDone)
+        await reader.cancel().catch(() => undefined);
+    if (!content.trim())
+        throw new Error(`${provider} returned an empty response`);
+    return {
+        provider,
+        model,
+        content,
+        reasoning: reasoning.trim() || undefined,
+        usage: responseUsage(usage),
+        performance: responsePerformance(timings, usage, performance.now() - startedAt),
+    };
+};

package/dist/src/generation/providers/llama-cpp.js

@@ -0,0 +1,97 @@
+import { effectiveModelDirectories, getGenerationConfig, generationPathExists } from '../config.js';
+import { findLlamaExecutables } from '../llama-install.js';
+import { getLlamaFailureContext, getManagedLlamaStatus, withLlamaServer, } from '../llama-process.js';
+import { discoverGgufModels, localGgufModelName } from '../model-discovery.js';
+import { openAiCompatibleChat, openAiCompatibleChatStream } from '../openai-compatible.js';
+export const createLlamaCppProvider = (fetchImpl = fetch) => ({
+    id: 'llama-cpp',
+    label: 'llama.cpp',
+    local: true,
+    async status() {
+        const config = await getGenerationConfig();
+        const executable = config.llamaCpp.executablePath
+            ? generationPathExists(config.llamaCpp.executablePath)
+                ? config.llamaCpp.executablePath
+                : undefined
+            : (await findLlamaExecutables())[0];
+        const available = Boolean(config.llamaCpp.baseUrl || executable);
+        return {
+            id: 'llama-cpp',
+            label: 'llama.cpp',
+            available,
+            local: true,
+            detail: config.llamaCpp.baseUrl
+                ? `External local server: ${config.llamaCpp.baseUrl}`
+                : executable
+                    ? `Executable: ${executable}`
+                    : 'llama-server not found; run npm run setup:llama or set its path.',
+        };
+    },
+    async listModels() {
+        const config = await getGenerationConfig();
+        if (config.llamaCpp.baseUrl) {
+            const response = await fetchImpl(`${config.llamaCpp.baseUrl}/v1/models`, {
+                signal: AbortSignal.timeout(10_000),
+            });
+            if (!response.ok)
+                throw new Error(`llama.cpp models request failed (${response.status})`);
+            const payload = (await response.json());
+            return (payload.data ?? [])
+                .filter((model) => Boolean(model.id))
+                .map((model) => ({
+                id: model.id,
+                name: model.id,
+                provider: 'llama-cpp',
+                loaded: true,
+            }));
+        }
+        const [models, runtime] = await Promise.all([
+            discoverGgufModels(effectiveModelDirectories(config.llamaCpp)),
+            Promise.resolve(getManagedLlamaStatus()),
+        ]);
+        return models.map((model) => ({
+            ...model,
+            loaded: runtime.state !== 'stopped' && runtime.model !== undefined && model.id === runtime.model,
+        }));
+    },
+    async chat(request, signal) {
+        const managed = !(await getGenerationConfig()).llamaCpp.baseUrl;
+        try {
+            const response = await withLlamaServer(request.model, (baseUrl) => openAiCompatibleChat({
+                provider: 'llama-cpp',
+                baseUrl,
+                request,
+                signal,
+                fetchImpl,
+            }));
+            return {
+                ...response,
+                model: managed ? localGgufModelName(request.model) : response.model,
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            const context = managed ? getLlamaFailureContext() : undefined;
+            throw new Error([`llama.cpp generation failed: ${message}`, context].filter(Boolean).join('\n\n'), { cause: error });
+        }
+    },
+    async chatStream(request, onDelta, signal) {
+        const managed = !(await getGenerationConfig()).llamaCpp.baseUrl;
+        const model = managed ? localGgufModelName(request.model) : request.model;
+        try {
+            const response = await withLlamaServer(request.model, (baseUrl) => openAiCompatibleChatStream({
+                provider: 'llama-cpp',
+                baseUrl,
+                request,
+                signal,
+                fetchImpl,
+            }, (delta) => onDelta({ ...delta, model: managed ? model : delta.model })));
+            return { ...response, model: managed ? model : response.model };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            const context = managed ? getLlamaFailureContext() : undefined;
+            throw new Error([`llama.cpp generation failed: ${message}`, context].filter(Boolean).join('\n\n'), { cause: error });
+        }
+    },
+});

package/dist/src/generation/providers/openrouter.js

@@ -0,0 +1,106 @@
+import { getGenerationConfig, getOpenRouterApiKey } from '../config.js';
+import { openAiCompatibleChat, openAiCompatibleChatStream } from '../openai-compatible.js';
+export const isFreeOpenRouterModel = (model) => {
+    if (model.id === 'openrouter/free' || model.id.endsWith(':free'))
+        return true;
+    const prompt = Number(model.promptPrice);
+    const completion = Number(model.completionPrice);
+    return (model.promptPrice !== undefined &&
+        model.promptPrice.trim() !== '' &&
+        model.completionPrice !== undefined &&
+        model.completionPrice.trim() !== '' &&
+        Number.isFinite(prompt) &&
+        Number.isFinite(completion) &&
+        prompt === 0 &&
+        completion === 0);
+};
+const routingPreferences = (enforceZdr, denyDataCollection) => {
+    const provider = {};
+    if (enforceZdr)
+        provider.zdr = true;
+    if (denyDataCollection)
+        provider.data_collection = 'deny';
+    return Object.keys(provider).length > 0 ? { provider } : undefined;
+};
+const openRouterCreatedAt = (value) => {
+    if (typeof value !== 'number' || !Number.isFinite(value))
+        return undefined;
+    const date = new Date(value * 1000);
+    return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
+};
+export const createOpenRouterProvider = (fetchImpl = fetch) => ({
+    id: 'openrouter',
+    label: 'OpenRouter',
+    local: false,
+    async status() {
+        const config = await getGenerationConfig();
+        return {
+            id: 'openrouter',
+            label: 'OpenRouter',
+            available: config.openRouter.apiKeyConfigured,
+            local: false,
+            detail: config.openRouter.apiKeyConfigured
+                ? 'API key configured; prompts leave this device.'
+                : 'Set OPENROUTER_API_KEY or a session key in Settings.',
+        };
+    },
+    async listModels(options = {}) {
+        const config = await getGenerationConfig();
+        const key = getOpenRouterApiKey();
+        if (!key)
+            throw new Error('OpenRouter API key is not configured');
+        const url = new URL(`${config.openRouter.baseUrl}/models`);
+        if (options.sort && options.sort !== 'default')
+            url.searchParams.set('sort', options.sort);
+        const response = await fetchImpl(url, {
+            headers: { Authorization: `Bearer ${key}`, 'User-Agent': 'ThreadShelf' },
+            signal: AbortSignal.timeout(30_000),
+        });
+        const payload = (await response.json().catch(() => ({})));
+        if (!response.ok) {
+            throw new Error(payload.error?.message || `OpenRouter models request failed (${response.status})`);
+        }
+        return (payload.data ?? [])
+            .filter((model) => Boolean(model.id))
+            .map((model) => ({
+            id: model.id,
+            name: model.name || model.id,
+            provider: 'openrouter',
+            contextLength: model.context_length,
+            createdAt: openRouterCreatedAt(model.created),
+            promptPrice: model.pricing?.prompt,
+            completionPrice: model.pricing?.completion,
+        }))
+            .filter((model) => !options.freeOnly || isFreeOpenRouterModel(model));
+    },
+    async chat(request, signal) {
+        const config = await getGenerationConfig();
+        const key = getOpenRouterApiKey();
+        if (!key)
+            throw new Error('OpenRouter API key is not configured');
+        return openAiCompatibleChat({
+            provider: 'openrouter',
+            baseUrl: config.openRouter.baseUrl,
+            apiKey: key,
+            request,
+            signal,
+            fetchImpl,
+            extraBody: routingPreferences(request.openRouterZdr ?? config.openRouter.enforceZdr, config.openRouter.denyDataCollection),
+        });
+    },
+    async chatStream(request, onDelta, signal) {
+        const config = await getGenerationConfig();
+        const key = getOpenRouterApiKey();
+        if (!key)
+            throw new Error('OpenRouter API key is not configured');
+        return openAiCompatibleChatStream({
+            provider: 'openrouter',
+            baseUrl: config.openRouter.baseUrl,
+            apiKey: key,
+            request,
+            signal,
+            fetchImpl,
+            extraBody: routingPreferences(request.openRouterZdr ?? config.openRouter.enforceZdr, config.openRouter.denyDataCollection),
+        }, onDelta);
+    },
+});