thumbgate 1.35.0 → 1.37.2

package/scripts/model-tier-router.js

@@ -32,11 +32,18 @@ function loadConfig() {
 const TIERS = {
   nano: { label: 'nano', costMultiplier: 0.1, maxContext: 32000 },
   mini: { label: 'mini', costMultiplier: 0.4, maxContext: 200000 },
+  // Cheap OpenAI-compatible cloud flagship (Qwen3.8-Max class) for steady
+  // high-output workloads. Inert until its endpoint env vars are configured.
+  bulkCloud: { label: 'bulk-cloud', costMultiplier: 0.25, maxContext: 1000000 },
   frontier: { label: 'frontier', costMultiplier: 1.0, maxContext: 1000000 },
   // Self-hosted open-source frontier (e.g. GLM 5.1). Zero marginal cost.
   localFrontier: { label: 'local-frontier', costMultiplier: 0.0, maxContext: 1000000 },
 };
 
+// Routing priority for task-type matching. Config-declared tiers missing from
+// this list are appended after, before the frontier fallback semantics apply.
+const TIER_ROUTING_ORDER = ['nano', 'mini', 'bulkCloud', 'frontier'];
+
 // ---------------------------------------------------------------------------
 // Task classification → tier mapping
 // ---------------------------------------------------------------------------
@@ -91,8 +98,10 @@ function classifyTask(task = {}) {
   // --- Normal tier routing by task type ---
 
   const tiers = config.tiers;
-  for (const tierName of ['nano', 'mini', 'frontier']) {
-    if (tiers[tierName].taskTypes.includes(type)) {
+  const declaredOrder = TIER_ROUTING_ORDER.filter((name) => tiers[name])
+    .concat(Object.keys(tiers).filter((name) => !TIER_ROUTING_ORDER.includes(name)));
+  for (const tierName of declaredOrder) {
+    if (Array.isArray(tiers[tierName].taskTypes) && tiers[tierName].taskTypes.includes(type)) {
       return {
         tier: tierName,
         reason: `task type "${type}" mapped to ${tierName}`,
@@ -269,6 +278,19 @@ class FrontierBudget {
   }
 }
 
+function isSwitchyardEnabled(task = {}, env = process.env) {
+  const flag = String(env.THUMBGATE_SWITCHYARD || '').trim().toLowerCase();
+  if (flag === '1' || flag === 'true' || flag === 'yes' || flag === 'on') return true;
+  const tags = Array.isArray(task.tags) ? task.tags.map((t) => String(t).toLowerCase()) : [];
+  return Boolean(
+    Array.isArray(task.steps) && task.steps.length > 0
+    || tags.includes('switchyard')
+    || tags.includes('multi-model')
+    || tags.includes('nemotron')
+    || tags.includes('always-on'),
+  );
+}
+
 function recommendExecutionPlan(task = {}, env = process.env) {
   const classification = classifyTask(task);
   const inference = recommendInferenceBackend(task, env);
@@ -280,7 +302,7 @@ function recommendExecutionPlan(task = {}, env = process.env) {
     ? 'localFrontier'
     : classification.tier;
 
-  return {
+  const plan = {
     architecture: 'risk-aware-model-routing',
     mixtureOfExperts: false,
     tier: effectiveTier,
@@ -295,6 +317,52 @@ function recommendExecutionPlan(task = {}, env = process.env) {
     indexCacheEnabled: inference.backend.indexCacheEnabled,
     reason: `${classification.reason}; ${inference.reason}`,
   };
+
+  // Optional NeMo Switchyard–style multi-model step plan (opt-in).
+  if (isSwitchyardEnabled(task, env)) {
+    try {
+      const {
+        routeAgentSteps,
+        buildAlwaysOnAgentPlan,
+      } = require('./switchyard-router');
+      const steps = Array.isArray(task.steps) && task.steps.length
+        ? task.steps
+        : buildAlwaysOnAgentPlan({
+          sensitive: task.privacyRoute === 'local' || task.sensitive,
+          highVolume: task.highVolume || task.costPriority === 'primary',
+          riskLevel: task.riskLevel,
+          actType: task.type,
+        });
+      const switchyard = routeAgentSteps(steps);
+      plan.switchyard = switchyard;
+      plan.multiModel = switchyard.multiModel;
+      plan.reason = `${plan.reason}; switchyard models=${switchyard.distinctModels.join(',')}`;
+    } catch {
+      // Switchyard optional; never break baseline routing.
+    }
+  }
+
+  // Optional dual-stack cost lane (Qwen volume / Claude quality). Opt-in via
+  // THUMBGATE_COST_ROUTE_QWEN=1 or task tags cost-sensitive|high-volume|bulk.
+  if (isCostRouteQwenEnabled(task, env)) {
+    try {
+      const { recommendCostQualitySplit } = require('./qwen38-max-cost-optimizer');
+      const split = recommendCostQualitySplit(task, {
+        useTokenPlanPromo: String(env.THUMBGATE_QWEN_TOKEN_PLAN_PROMO || '').toLowerCase() === '1'
+          || String(env.THUMBGATE_QWEN_TOKEN_PLAN_PROMO || '').toLowerCase() === 'true',
+        localModel: env.THUMBGATE_LOCAL_MODEL || null,
+      });
+      plan.costQualitySplit = split;
+      plan.costLane = split.lane;
+      plan.preferredProvider = split.primaryProvider;
+      plan.preferredModel = split.primaryModel;
+      plan.reason = `${plan.reason}; cost-lane=${split.lane}→${split.primaryProvider}/${split.primaryModel}`;
+    } catch {
+      // Cost module optional at runtime; never fail routing.
+    }
+  }
+
+  return plan;
 }
 
 // ---------------------------------------------------------------------------
@@ -371,23 +439,46 @@ function resolveGenerationTarget(plan, task, tierConfig, options, env) {
 
   const model = modelOverride || tierConfig.modelId;
   if (!model) throw new Error(`missing modelId for tier "${plan.tier}"`);
-  return {
+  const target = {
     model,
     provider: providerOverride || tierConfig.provider || inferProvider(model, plan.tier),
   };
+  if (tierConfig.endpoint && typeof tierConfig.endpoint === 'object') {
+    const { baseUrlEnv, apiKeyEnv } = tierConfig.endpoint;
+    const baseUrl = baseUrlEnv ? env[baseUrlEnv] : undefined;
+    const apiKey = apiKeyEnv ? env[apiKeyEnv] : undefined;
+    if (!baseUrl || !apiKey) {
+      const missing = [!baseUrl && baseUrlEnv, !apiKey && apiKeyEnv].filter(Boolean).join(', ');
+      throw new Error(`tier "${plan.tier}" requires endpoint configuration; set ${missing}`);
+    }
+    target.endpoint = { baseUrl, apiKey };
+  }
+  return target;
+}
+
+/**
+ * Estimate the USD cost of a generation for a tier that declares
+ * pricingUsdPerMTok in config. Returns null when pricing is not declared.
+ */
+function estimateTierCostUsd(tierName, inputTokens, outputTokens, config = loadConfig()) {
+  const pricing = config?.tiers?.[tierName]?.pricingUsdPerMTok;
+  if (!pricing || !Number.isFinite(pricing.input) || !Number.isFinite(pricing.output)) return null;
+  const inTok = Number.isFinite(inputTokens) ? Math.max(0, inputTokens) : 0;
+  const outTok = Number.isFinite(outputTokens) ? Math.max(0, outputTokens) : 0;
+  return (inTok * pricing.input + outTok * pricing.output) / 1_000_000;
 }
 
 function createOpenAiCompatibleAdapter(options = {}) {
   const env = options.env || process.env;
   const fetchImpl = options.fetchImpl || global.fetch;
 
-  return async function openAiCompatibleAdapter({ request, model, provider }) {
+  return async function openAiCompatibleAdapter({ request, model, provider, endpoint }) {
     if (typeof fetchImpl !== 'function') throw new Error('fetch is unavailable');
     const local = provider === 'openai-compatible';
-    const baseUrl = local
-      ? env.THUMBGATE_LOCAL_MODEL_BASE_URL
-      : (env.OPENAI_BASE_URL || 'https://api.openai.com/v1');
-    const apiKey = local ? env.THUMBGATE_LOCAL_MODEL_API_KEY : env.OPENAI_API_KEY;
+    const baseUrl = endpoint?.baseUrl
+      || (local ? env.THUMBGATE_LOCAL_MODEL_BASE_URL : (env.OPENAI_BASE_URL || 'https://api.openai.com/v1'));
+    const apiKey = endpoint?.apiKey
+      || (local ? env.THUMBGATE_LOCAL_MODEL_API_KEY : env.OPENAI_API_KEY);
     if (!baseUrl) throw new Error('THUMBGATE_LOCAL_MODEL_BASE_URL is required for local routing');
     if (!local && !apiKey) throw new Error('OPENAI_API_KEY is required for OpenAI routing');
 
@@ -443,38 +534,56 @@ async function executeRoutedGeneration(task = {}, request = {}, options = {}) {
   const plan = recommendExecutionPlan(task, env);
   const tierConfig = config.tiers[plan.tier];
   if (!tierConfig) throw new Error(`missing configuration for tier "${plan.tier}"`);
-  const { model, provider } = resolveGenerationTarget(plan, task, tierConfig, options, env);
-  const adapters = options.adapters || createDefaultGenerationAdapters({ env, fetchImpl: options.fetchImpl });
-  const adapter = adapters[plan.tier] || adapters[provider];
-  if (typeof adapter !== 'function') throw new Error(`no generation adapter registered for provider "${provider}"`);
-
   const now = options.now || (() => Date.now());
   const startedAt = now();
+  // Resolution can fail (e.g. an unconfigured tier endpoint), so the base event
+  // starts from tier config defaults and target resolution happens inside the
+  // telemetry try block — misconfiguration emits an error event, not silence.
   const baseEvent = {
     timestamp: new Date().toISOString(),
     architecture: plan.architecture,
     taskType: task.type || 'unknown',
     riskLevel: task.riskLevel || 'unspecified',
     tier: plan.tier,
-    provider,
-    model,
+    provider: tierConfig.provider || null,
+    model: tierConfig.modelId || null,
     escalated: plan.escalated,
     routeReason: plan.reason,
   };
 
   try {
-    const raw = await adapter({ task, request, plan, model, provider });
+    const { model, provider, endpoint } = resolveGenerationTarget(plan, task, tierConfig, options, env);
+    baseEvent.provider = provider;
+    baseEvent.model = model;
+    const adapters = options.adapters || createDefaultGenerationAdapters({ env, fetchImpl: options.fetchImpl });
+    const adapter = adapters[plan.tier] || adapters[provider];
+    if (typeof adapter !== 'function') throw new Error(`no generation adapter registered for provider "${provider}"`);
+    const raw = await adapter({ task, request, plan, model, provider, endpoint });
     const result = normalizeGenerationResult(raw, { model, provider });
+    const inputTokens = Number(result.usage?.input_tokens || result.usage?.prompt_tokens || 0) || null;
+    const outputTokens = Number(result.usage?.output_tokens || result.usage?.completion_tokens || 0) || null;
+    // When the adapter reports no cost but the tier declares per-token pricing,
+    // derive real cost from usage so telemetry and holdout cost math stay live.
+    // Never derive when usage is entirely unmeasured (an unmeasured request is
+    // unknown-cost, not free) or when execution actually ran on a local backend
+    // (tier pricing describes the cloud endpoint, not local inference).
+    let costCents = result.costCents;
+    const usageMeasured = inputTokens !== null || outputTokens !== null;
+    if ((costCents === null || costCents === undefined) && usageMeasured && plan.providerMode !== 'local') {
+      const estimatedUsd = estimateTierCostUsd(plan.tier, inputTokens || 0, outputTokens || 0, config);
+      costCents = estimatedUsd === null ? null : Number((estimatedUsd * 100).toFixed(4));
+    }
+    if (costCents === undefined) costCents = null;
     const event = {
       ...baseEvent,
       latencyMs: Math.max(0, now() - startedAt),
-      inputTokens: Number(result.usage?.input_tokens || result.usage?.prompt_tokens || 0) || null,
-      outputTokens: Number(result.usage?.output_tokens || result.usage?.completion_tokens || 0) || null,
-      costCents: result.costCents,
+      inputTokens,
+      outputTokens,
+      costCents,
       outcome: 'success',
     };
     if (options.telemetrySink) await options.telemetrySink(event);
-    return { ...result, route: plan, telemetry: event };
+    return { ...result, costCents, route: plan, telemetry: event };
   } catch (error) {
     const event = {
       ...baseEvent,
@@ -564,12 +673,30 @@ async function evaluateRoutingHoldout(cases, options = {}) {
 // Exports
 // ---------------------------------------------------------------------------
 
+function isCostRouteQwenEnabled(task = {}, env = process.env) {
+  const flag = String(env.THUMBGATE_COST_ROUTE_QWEN || '').trim().toLowerCase();
+  if (flag === '1' || flag === 'true' || flag === 'yes' || flag === 'on') return true;
+  const tags = Array.isArray(task.tags) ? task.tags.map((t) => String(t).toLowerCase()) : [];
+  return Boolean(
+    task.costPriority === 'primary'
+    || task.highVolume
+    || tags.includes('cost-sensitive')
+    || tags.includes('high-volume')
+    || tags.includes('bulk')
+    || tags.includes('qwen-volume'),
+  );
+}
+
 module.exports = {
+  isCostRouteQwenEnabled,
   TIERS,
+  TIER_ROUTING_ORDER,
   classifyTask,
+  estimateTierCostUsd,
   shouldEscalate,
   FrontierBudget,
   recommendExecutionPlan,
+  isSwitchyardEnabled,
   inferProvider,
   normalizeGenerationResult,
   resolveGenerationTarget,