thumbgate 1.27.14 → 1.27.16

package/scripts/llm-client.js

@@ -16,6 +16,8 @@ const DEFAULT_MAX_TOKENS = 1024;
 const DEFAULT_CACHE_TTL = '5m';
 const DEFAULT_ZAI_BASE_URL = 'https://api.z.ai/api/coding/paas/v4';
 const DEFAULT_ZAI_MODEL = MODELS.ZAI_CODING;
+const DEFAULT_OMLX_BASE_URL = 'http://127.0.0.1:8000/v1';
+const DEFAULT_OMLX_MODEL = 'qwen3-coder-next';
 
 let _anthropicClient = null;
 let _geminiClient = null;
@@ -32,14 +34,40 @@ function getZaiModel(env = process.env) {
   return env.ZAI_API_MODEL || env.THUMBGATE_ZAI_MODEL || DEFAULT_ZAI_MODEL;
 }
 
+function getOmlxApiKey(env = process.env) {
+  return env.OMLX_API_KEY || env.THUMBGATE_OMLX_API_KEY || '';
+}
+
+function getOmlxBaseUrl(env = process.env) {
+  return (env.OMLX_BASE_URL || env.THUMBGATE_OMLX_BASE_URL || DEFAULT_OMLX_BASE_URL)
+    .replace(/\/chat\/completions\/?$/i, '')
+    .replace(/\/+$/, '');
+}
+
+function getOmlxModel(env = process.env) {
+  return env.OMLX_MODEL || env.THUMBGATE_OMLX_MODEL || DEFAULT_OMLX_MODEL;
+}
+
 function hasZaiApiKey(env = process.env) {
   return Boolean(getZaiApiKey(env));
 }
 
+function hasOmlxConfig(env = process.env) {
+  return env.THUMBGATE_LLM_PROVIDER === 'omlx'
+    || env.LLM_PROVIDER === 'omlx'
+    || env.THUMBGATE_OMLX_ENABLED === '1'
+    || env.OMLX_ENABLED === '1'
+    || Boolean(env.OMLX_BASE_URL || env.THUMBGATE_OMLX_BASE_URL);
+}
+
 function isZaiProvider(provider = '') {
   return /^(zai|z\.ai|glm|glm-coding)$/i.test(String(provider || '').trim());
 }
 
+function isOmlxProvider(provider = '') {
+  return /^(omlx|local-omlx|mlx-local)$/i.test(String(provider || '').trim());
+}
+
 function isZaiModel(model = '') {
   return /^(glm|zai|z\.ai)/i.test(String(model || '').trim());
 }
@@ -47,13 +75,14 @@ function isZaiModel(model = '') {
 function isAvailable(provider = '') {
   const normalizedProvider = String(provider || '').trim().toLowerCase();
   if (isZaiProvider(normalizedProvider)) return hasZaiApiKey();
+  if (isOmlxProvider(normalizedProvider)) return hasOmlxConfig();
   if (normalizedProvider === 'anthropic' || normalizedProvider === 'claude') {
     return Boolean(process.env.ANTHROPIC_API_KEY);
   }
   if (normalizedProvider === 'gemini' || normalizedProvider === 'vertex') {
     return Boolean(process.env.GEMINI_API_KEY || process.env.VERTEX_PROJECT_ID);
   }
-  return Boolean(hasZaiApiKey() || process.env.ANTHROPIC_API_KEY || process.env.GEMINI_API_KEY || process.env.VERTEX_PROJECT_ID);
+  return Boolean(hasOmlxConfig() || hasZaiApiKey() || process.env.ANTHROPIC_API_KEY || process.env.GEMINI_API_KEY || process.env.VERTEX_PROJECT_ID);
 }
 
 function getClient() {
@@ -278,6 +307,51 @@ async function callZaiInternal(options = {}) {
   }
 }
 
+async function callOmlxInternal(options = {}) {
+  if (!hasOmlxConfig()) return null;
+  if (typeof fetch !== 'function') return null;
+
+  const baseUrl = getOmlxBaseUrl();
+  const request = buildOpenAiCompatibleRequest({
+    ...options,
+    model: options.model || getOmlxModel(),
+  });
+  const apiKey = getOmlxApiKey();
+  const headers = {
+    'Content-Type': 'application/json',
+  };
+  if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+
+  try {
+    const response = await runStep('llm.callOmlx', {
+      retries: 1,
+      logger: (msg) => console.warn(msg),
+    }, async () => fetch(`${baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(request),
+    }));
+
+    if (!response || !response.ok) {
+      throw new Error(`oMLX request failed with status ${response?.status || 'unknown'}`);
+    }
+
+    const json = await response.json();
+    const text = stripCodeFences(json?.choices?.[0]?.message?.content || '');
+    return {
+      text,
+      usage: json?.usage || null,
+      stopReason: json?.choices?.[0]?.finish_reason || null,
+      id: json?.id || null,
+      model: json?.model || request.model,
+      provider: 'omlx',
+    };
+  } catch (err) {
+    console.error('oMLX execution error:', err?.message || err);
+    return null;
+  }
+}
+
 async function callGeminiInternal(options = {}) {
   const env = process.env;
   const { detectInferenceBackend } = require('./local-model-profile');
@@ -362,6 +436,9 @@ async function callClaudeInternal(options = {}) {
   const modelName = options.model || '';
   const provider = options.provider || process.env.THUMBGATE_LLM_PROVIDER || process.env.LLM_PROVIDER || '';
   const requestedZai = isZaiProvider(provider) || isZaiModel(modelName);
+  if (isOmlxProvider(provider)) {
+    return callOmlxInternal(options);
+  }
   if (requestedZai || (!process.env.ANTHROPIC_API_KEY && hasZaiApiKey() && !modelName.startsWith('gemini') && !modelName.startsWith('vertex'))) {
     const zaiOptions = requestedZai ? options : { ...options, model: undefined };
     return callZaiInternal(zaiOptions);
@@ -419,6 +496,7 @@ async function callClaudeJson(options = {}) {
       stopReason: result.stopReason,
       id: result.id,
       model: result.model,
+      provider: result.provider || 'anthropic',
     };
   }
 
@@ -453,12 +531,42 @@ async function callZaiJson(options = {}) {
   return parsed;
 }
 
+async function callOmlx(options = {}) {
+  const result = await callOmlxInternal(options);
+  if (!result) return null;
+  return options.returnMetadata ? result : result.text;
+}
+
+async function callOmlxJson(options = {}) {
+  const result = await callOmlxInternal(options);
+  if (!result) return null;
+
+  const parsed = parseClaudeJson(result.text);
+  if (parsed === null) return null;
+
+  if (options.returnMetadata) {
+    return {
+      parsed,
+      text: result.text,
+      usage: result.usage,
+      stopReason: result.stopReason,
+      id: result.id,
+      model: result.model,
+      provider: result.provider,
+    };
+  }
+
+  return parsed;
+}
+
 module.exports = {
   isAvailable,
   callClaude,
   callClaudeJson,
   callZai,
   callZaiJson,
+  callOmlx,
+  callOmlxJson,
   stripCodeFences,
   parseClaudeJson,
   normalizeCacheOptions,
@@ -468,6 +576,11 @@ module.exports = {
   getZaiApiKey,
   getZaiBaseUrl,
   getZaiModel,
+  getOmlxApiKey,
+  getOmlxBaseUrl,
+  getOmlxModel,
   MODELS,
   DEFAULT_ZAI_BASE_URL,
+  DEFAULT_OMLX_BASE_URL,
+  DEFAULT_OMLX_MODEL,
 };

package/scripts/omlx-smoke.js

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+function loadDotEnv(filePath = path.join(process.cwd(), '.env')) {
+  if (!fs.existsSync(filePath)) return false;
+  const text = fs.readFileSync(filePath, 'utf8');
+  for (const line of text.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eq = trimmed.indexOf('=');
+    if (eq <= 0) continue;
+    const key = trimmed.slice(0, eq).trim();
+    const rawValue = trimmed.slice(eq + 1).trim();
+    if (!key || process.env[key]) continue;
+    process.env[key] = rawValue.replace(/^['"]|['"]$/g, '');
+  }
+  return true;
+}
+
+function parseArgs(argv = []) {
+  const out = {
+    json: argv.includes('--json'),
+    requireLive: argv.includes('--require-live'),
+    chat: argv.includes('--chat'),
+    baseUrl: null,
+    model: null,
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    if (argv[i] === '--base-url') out.baseUrl = argv[++i];
+    if (argv[i] === '--model') out.model = argv[++i];
+  }
+
+  return out;
+}
+
+function normalizeBaseUrl(baseUrl) {
+  return String(baseUrl || '')
+    .replace(/\/chat\/completions\/?$/i, '')
+    .replace(/\/models\/?$/i, '')
+    .replace(/\/+$/, '');
+}
+
+function extractModels(payload) {
+  const data = Array.isArray(payload?.data) ? payload.data : [];
+  return data
+    .map((entry) => {
+      if (typeof entry === 'string') return entry;
+      if (entry && typeof entry === 'object') return entry.id || entry.name || null;
+      return null;
+    })
+    .filter(Boolean);
+}
+
+async function fetchModels(baseUrl, fetchImpl = global.fetch) {
+  if (typeof fetchImpl !== 'function') {
+    return { ok: false, status: 'fetch_unavailable', models: [] };
+  }
+
+  const url = `${normalizeBaseUrl(baseUrl)}/models`;
+  try {
+    const response = await fetchImpl(url, { method: 'GET' });
+    if (!response || !response.ok) {
+      return {
+        ok: false,
+        status: 'models_endpoint_failed',
+        httpStatus: response?.status || null,
+        url,
+        models: [],
+      };
+    }
+    const payload = await response.json();
+    return {
+      ok: true,
+      status: 'models_ready',
+      httpStatus: response.status || 200,
+      url,
+      models: extractModels(payload),
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      status: 'models_endpoint_error',
+      url,
+      message: error?.message || String(error),
+      models: [],
+    };
+  }
+}
+
+async function run(options = {}) {
+  loadDotEnv();
+  const {
+    callOmlx,
+    getOmlxBaseUrl,
+    getOmlxModel,
+  } = require('./llm-client');
+
+  const baseUrl = normalizeBaseUrl(options.baseUrl || getOmlxBaseUrl());
+  const modelsResult = await fetchModels(baseUrl, options.fetchImpl || global.fetch);
+  const configuredModel = options.model || getOmlxModel();
+  const selectedModel = options.model || modelsResult.models[0] || configuredModel;
+  const result = {
+    ok: modelsResult.ok,
+    status: modelsResult.status,
+    provider: 'omlx',
+    baseUrl,
+    model: selectedModel,
+    models: modelsResult.models,
+    modelsUrl: modelsResult.url,
+    httpStatus: modelsResult.httpStatus || null,
+  };
+
+  if (!modelsResult.ok) {
+    result.message = modelsResult.message || 'oMLX /v1/models is not reachable';
+    if (options.requireLive) process.exitCode = 1;
+    return result;
+  }
+
+  if (!options.chat) {
+    result.status = modelsResult.models.length > 0 ? 'models_ready' : 'models_empty';
+    result.ok = !options.requireLive || modelsResult.models.length > 0;
+    if (!result.ok) process.exitCode = 1;
+    return result;
+  }
+
+  process.env.THUMBGATE_OMLX_ENABLED = '1';
+  process.env.THUMBGATE_OMLX_BASE_URL = baseUrl;
+  process.env.THUMBGATE_OMLX_MODEL = selectedModel;
+
+  const chat = await callOmlx({
+    systemPrompt: 'You are a local inference smoke test.',
+    userPrompt: 'Say that ThumbGate local oMLX inference is reachable.',
+    maxTokens: 64,
+    temperature: 0,
+    returnMetadata: true,
+  });
+
+  if (!chat || typeof chat.text !== 'string' || chat.text.trim().length < 1) {
+    process.exitCode = 1;
+    return {
+      ...result,
+      ok: false,
+      status: 'chat_failed',
+      message: 'oMLX /v1/chat/completions did not return assistant text',
+    };
+  }
+
+  return {
+    ...result,
+    ok: true,
+    status: 'ready',
+    model: chat.model || selectedModel,
+    textPreview: chat.text.slice(0, 200),
+    usage: chat.usage || null,
+  };
+}
+
+async function main(argv = process.argv.slice(2)) {
+  const options = parseArgs(argv);
+  const result = await run(options);
+  if (options.json) {
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  console.log(`oMLX smoke status: ${result.status}`);
+  console.log(`Provider: ${result.provider}`);
+  console.log(`Base URL: ${result.baseUrl}`);
+  console.log(`Model: ${result.model}`);
+  console.log(`Models: ${result.models.length}`);
+  console.log(`Result: ${result.ok ? 'ready' : result.message || 'not ready'}`);
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error?.message || error);
+    process.exit(1);
+  });
+}
+
+module.exports = {
+  loadDotEnv,
+  parseArgs,
+  normalizeBaseUrl,
+  extractModels,
+  fetchModels,
+  run,
+};